s3_website 2.1.6 → 2.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07675e2836114a62d64ba9c4f3cb1d468979def7
-  data.tar.gz: 79ec43e71d28fa9eb6852883f3845a91c71adba5
+  metadata.gz: 8a50d9894a8f340bff511731fe4934a413e324a2
+  data.tar.gz: 23008693b34025381965ff5721796c78d63a5b72
 SHA512:
-  metadata.gz: 5b8da0a144df3b8e6a9af48457927f54e9991afd9d81a7eb00ab215c754d72905727dc14298bbeade33190b2a88c14f27fca0e2b28cfbc11bdaaa13e8d60124c
-  data.tar.gz: 83b3cc06196da7997bcd5f0d051c70e08b7ccc0d224c04070adc14e675788fc15c5e90c2d503489d46f854057373c5757969a579315c37059380bff5d9d267f2
+  metadata.gz: 98f1cb95e395dc00cc47f71af25cd2ac40e75e08dd7299e89df85d403e0179abbfcec2ae1c0b85caf9b1235fcd9a4fee25116d7bc7a6d511209ff8bb3504c6ba
+  data.tar.gz: 396c0a1d1a4f9a45baa6365a0041955d169b391384c243267092faa9a57be3f1d65d0ed8a7d1fc555b63ad5f07d19416219f3dc4968c5102c52aa1c04dbb701f

data/changelog.md

@@ -2,6 +2,12 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
+## 2.1.7
+
+* Remove local db
+
+  It turned out to be too complex to maintain
+
 ## 2.1.6
 
 * Automatically add slash to redirects if needed
@@ -40,9 +46,6 @@ This project uses [Semantic Versioning](http://semver.org).
 
 * Faster uploads for extra large sites
 
-   Use a local database for calculating diffs. This removes the need to read all
-   the files of the website, when you call the `s3_website push` command.
-
    Use proper multithreading with JVM threads.
 
 * Simulate deployments with `push --dry-run`

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '2.1.6'
+  VERSION = '2.1.7'
 end

data/resources/s3_website.jar.md5

@@ -1 +1 @@
-103852dae1cdc7b6e85c0623ad526beb
+a0ae3ccc855ccb2b3488da03422f492a

data/src/main/scala/s3/website/Diff.scala

@@ -5,30 +5,18 @@ import s3.website.Ruby.rubyRegexMatches
 import scala.concurrent.{ExecutionContextExecutor, Future}
 import scala.util.{Failure, Success, Try}
 import java.io.File
-import org.apache.commons.io.FileUtils._
-import org.apache.commons.codec.digest.DigestUtils._
-import scala.io.Source
-import s3.website.Diff.LocalFileDatabase.resolveDiffAgainstLocalDb
-import s3.website.Diff.UploadBatch
-
-case class Diff(
-  unchanged: Future[Either[ErrorReport, Seq[S3Key]]],
-  uploads: Seq[UploadBatch],
-  persistenceError: Future[Option[ErrorReport]]
-)
 
 object Diff {
 
-  type UploadBatch = Future[Either[ErrorReport, Seq[Upload]]]
+  type FutureUploads = Future[Either[ErrorReport, Seq[Upload]]]
 
   def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
-                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
-    if (LocalFileDatabase.hasRecords) resolveDiffAgainstLocalDb(s3FilesFuture)
-    else resolveDiffAgainstGetBucketResponse(s3FilesFuture)
+                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): FutureUploads =
+    resolveDiffAgainstGetBucketResponse(s3FilesFuture)
 
   private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
-                                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
-    val diffAgainstS3 = s3FilesFuture.map { errorOrS3Files =>
+                                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): FutureUploads =
+    s3FilesFuture.map { errorOrS3Files =>
       errorOrS3Files.right.flatMap { s3Files =>
         Try {
           val s3KeyIndex = s3Files.map(_.s3Key).toSet
@@ -37,62 +25,30 @@ object Diff {
           val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
           val isChangedOnS3 = (upload: Upload) => !(s3Md5Index contains upload.md5.get)
           val newUploads = siteFiles collect {
-            case file if !existsOnS3(file) => Upload(file, NewFile, reasonForUpload = "the file is missing from S3")
+            case file if !existsOnS3(file) => Upload(file, NewFile)
           }
           val changedUploads = siteFiles collect {
-            case file if existsOnS3(file) => Upload(file, FileUpdate, reasonForUpload = "the S3 bucket has different contents for this file")
+            case file if existsOnS3(file) => Upload(file, FileUpdate)
           } filter isChangedOnS3
-          val unchangedFiles = {
-            val newOrChangedFiles = (changedUploads ++ newUploads).map(_.originalFile).toSet
-            siteFiles.filterNot(f => newOrChangedFiles contains f)
-          }
-          val recordsAndUploads: Seq[Either[DbRecord, Upload]] = unchangedFiles.map {
-            f => Left(DbRecord(f))
-          } ++ (changedUploads ++ newUploads).map {
-            Right(_)
-          }
-          LocalFileDatabase persist recordsAndUploads
-          recordsAndUploads
+          newUploads ++ changedUploads
         } match {
           case Success(ok) => Right(ok)
           case Failure(err) => Left(ErrorReport(err))
         }
       }
     }
-    def collectResult[B](pf: PartialFunction[Either[DbRecord, Upload],B]) =
-      diffAgainstS3.map { errorOrDiffSource =>
-        errorOrDiffSource.right map (_ collect pf)
-      }
-    val unchanged = collectResult {
-      case Left(dbRecord) => dbRecord.s3Key
-    }
-    val uploads: UploadBatch = collectResult {
-      case Right(upload) => upload
-    }
-    Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
-  }
 
-  def resolveDeletes(diff: Diff, s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
-                    (implicit config: Config, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = {
-    val localKeys = for {
-      errorOrUnchanged <- diff.unchanged
-      errorsOrChanges  <- Future.sequence(diff.uploads)
-    } yield
-      errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
-        for {
-          mem <- memo.right
-          keysToDelete <- errorOrChanges.right
-        } yield mem ++ keysToDelete.map(_.s3Key)
-      }
+  def resolveDeletes(s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
+                    (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = {
+    val localS3Keys = Files.listSiteFiles.map(site resolveS3Key)
 
-    s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
+    s3Files map { s3Files: Either[ErrorReport, Seq[S3File]] =>
       for {
-        localS3Keys <- errorOrLocalKeys.right
         remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
       } yield {
         val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet
         remoteS3Keys filterNot { s3Key =>
-          val ignoreOnServer = config.ignore_on_server.exists(_.fold(
+          val ignoreOnServer = site.config.ignore_on_server.exists(_.fold(
             (ignoreRegex: String) => rubyRegexMatches(s3Key, ignoreRegex),
             (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3Key, _))
           ))
@@ -100,203 +56,6 @@ object Diff {
           (keysToRetain contains s3Key) || ignoreOnServer
         }
       }
-    }.tupled
-  }
-
-  object LocalFileDatabase {
-    def hasRecords(implicit site: Site, logger: Logger) =
-      (for {
-        dbFile <- getOrCreateDbFile
-        databaseIndices <- loadDbFromFile(dbFile)
-      } yield databaseIndices.fullIndex.headOption.isDefined) getOrElse false
-
-    def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
-                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
-      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, Upload]]] =
-        (for {
-          dbFile <- getOrCreateDbFile
-          databaseIndices <- loadDbFromFile(dbFile)
-        } yield {
-          val siteFiles = Files.listSiteFiles
-          val recordsOrUploads = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, Upload]]) { (recordsOrUps, file) =>
-            val truncatedKey = TruncatedDbRecord(file)
-            val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
-            if (fileIsUnchanged)
-              recordsOrUps :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
-            else {
-              val isUpdate = databaseIndices.s3KeyIndex contains truncatedKey.s3Key
-
-              val uploadType =
-                if (isUpdate) FileUpdate
-                else NewFile
-              recordsOrUps :+ Right(Upload(file, uploadType, reasonForUpload(truncatedKey, databaseIndices, isUpdate)))
-            }
-          }
-          logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrUploads count (_.isRight)} are new or changed")
-          recordsOrUploads
-        }) match {
-          case Success(ok) => Right(ok)
-          case Failure(err) => Left(ErrorReport(err))
-        }
-
-      localDiff.right map { localDiffResult =>
-        val unchangedAccordingToLocalDiff = localDiffResult collect {
-          case Left(f) => f
-        }
-
-        val uploadsAccordingToLocalDiff = localDiffResult collect {
-          case Right(f) => f
-        }
-
-        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[Upload]]] = s3FilesFuture.map { errorOrS3Files =>
-          for (s3Files <- errorOrS3Files.right) yield {
-            val remoteS3Keys = s3Files.map(_.s3Key).toSet
-            val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
-            val localMd5 = unchangedAccordingToLocalDiff.map(_.uploadFileMd5).toSet
-            def isChangedOnS3(s3File: S3File) = (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
-            val changedOnS3 = s3Files collect {
-              case s3File if isChangedOnS3(s3File) =>
-                Upload(site resolveFile s3File, FileUpdate, reasonForUpload = "someone else has modified the file on the S3 bucket")
-            }
-            val missingFromS3 = localS3Keys collect {
-              case localS3Key if !(remoteS3Keys contains localS3Key) =>
-                Upload(site resolveFile localS3Key, NewFile, reasonForUpload = "someone else has removed the file from the S3 bucket")
-
-            }
-            changedOnS3 ++ missingFromS3
-          }
-        }
-
-        val errorOrDiffAgainstS3 =
-          changesMissedByLocalDiff map { errorOrUploads =>
-            errorOrUploads.right map { uploadsMissedByLocalDiff =>
-              val uploadsS3KeyIndex = uploadsMissedByLocalDiff.map(_.s3Key).toSet
-              val unchangedAccordingToLocalAndS3Diff = unchangedAccordingToLocalDiff.filterNot(uploadsS3KeyIndex contains _.s3Key)
-              (unchangedAccordingToLocalAndS3Diff, uploadsMissedByLocalDiff)
-            }
-          }
-
-        val unchangedFilesFinal = errorOrDiffAgainstS3 map {
-          _ fold (
-            (error: ErrorReport) => Left(error),
-            (syncResult: (Seq[DbRecord], Seq[Upload])) => Right(syncResult._1)
-          )
-        }
-
-        val uploadsAccordingToS3Diff = errorOrDiffAgainstS3.map {
-          _ fold (
-            (error: ErrorReport) => Left(error),
-            (syncResult: (Seq[DbRecord], Seq[Upload])) => Right(syncResult._2)
-          )
-        }
-        val persistenceError: Future[Either[ErrorReport, _]] = for {
-          unchanged <- unchangedFilesFinal
-          uploads <- uploadsAccordingToS3Diff
-        } yield
-          for {
-            records1 <- unchanged.right
-            records2 <- uploads.right
-          } yield
-            persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsAccordingToLocalDiff.map(Right(_))) match {
-              case Success(_)   => Unit
-              case Failure(err) => ErrorReport(err)
-            }
-        Diff(
-          unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
-          uploads = Future(Right(uploadsAccordingToLocalDiff)) :: uploadsAccordingToS3Diff :: Nil,
-          persistenceError = persistenceError map (_.left.toOption)
-        )
-      }
     }
-
-    private def reasonForUpload(truncatedKey: TruncatedDbRecord, databaseIndices: DbIndices, isUpdate: Boolean) = {
-      if (isUpdate) {
-        val lengthChanged = !(databaseIndices.fileLenghtIndex contains truncatedKey.fileLength)
-        val mtimeChanged = !(databaseIndices.lastModifiedIndex contains truncatedKey.fileModified)
-        if (mtimeChanged && lengthChanged)
-          "file mtime and length have changed"
-        else if (lengthChanged)
-          "file length has changed"
-        else if (mtimeChanged)
-          "file mtime has changed"
-        else
-          "programmer error: faulty logic in inferring the reason for upload"
-      }
-      else "file is new"
-    }
-
-    private def getOrCreateDbFile(implicit site: Site, logger: Logger) =
-      Try {
-        val dbFile = new File(getTempDirectory, "s3_website_local_db_" + sha256Hex(site.rootDirectory))
-        if (!dbFile.exists()) logger.debug("Creating a new database in " + dbFile.getName)
-        dbFile.createNewFile()
-        dbFile
-      }
-    
-    case class DbIndices(
-      s3KeyIndex:        Set[S3Key],
-      fileLenghtIndex:   Set[Long],
-      lastModifiedIndex: Set[Long],
-      truncatedIndex:    Set[TruncatedDbRecord],
-      fullIndex:         Set[DbRecord]
-    ) 
-
-    private def loadDbFromFile(databaseFile: File)(implicit site: Site, logger: Logger): Try[DbIndices] =
-      Try {
-        // record format: "s3Key(file.path)|length(file)|mtime(file)|md5Hex(file.encoded)"
-        val RecordRegex = "(.*?)\\|(\\d+)\\|(\\d+)\\|([a-f0-9]{32})".r
-        val fullIndex = Source
-          .fromFile(databaseFile, "utf-8")
-          .getLines()
-          .toStream
-          .map {
-            case RecordRegex(s3Key, fileLength, fileModified, md5) =>
-              DbRecord(s3Key, fileLength.toLong, fileModified.toLong, md5)
-          }
-          .toSet
-        DbIndices(
-          s3KeyIndex = fullIndex map (_.s3Key), 
-          truncatedIndex = fullIndex map (TruncatedDbRecord(_)),
-          fileLenghtIndex = fullIndex map (_.fileLength),
-          lastModifiedIndex = fullIndex map (_.fileModified),
-          fullIndex = fullIndex
-        )
-      }
-
-    def persist(recordsOrUploads: Seq[Either[DbRecord, Upload]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, Upload]]] =
-      getOrCreateDbFile flatMap { dbFile =>
-        Try {
-          val dbFileContents = recordsOrUploads.map { recordOrUpload =>
-            val record: DbRecord = recordOrUpload fold(
-              record => record,
-              upload => DbRecord(upload.s3Key, upload.originalFile.length, upload.originalFile.lastModified, upload.md5.get)
-              )
-            record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
-          } mkString "\n"
-
-          write(dbFile, dbFileContents)
-          recordsOrUploads
-        }
-      }
-  }
-
-  case class TruncatedDbRecord(s3Key: String, fileLength: Long, fileModified: Long)
-
-  object TruncatedDbRecord {
-    def apply(dbRecord: DbRecord): TruncatedDbRecord = TruncatedDbRecord(dbRecord.s3Key, dbRecord.fileLength, dbRecord.fileModified)
-    
-    def apply(file: File)(implicit site: Site): TruncatedDbRecord = TruncatedDbRecord(site resolveS3Key file, file.length, file.lastModified)
-  }
-
-  /**
-   * @param uploadFileMd5 if the file is gzipped, this checksum should be calculated on the gzipped file, not the original file
-   */
-  case class DbRecord(s3Key: String, fileLength: Long, fileModified: Long, uploadFileMd5: MD5) {
-    lazy val truncated = TruncatedDbRecord(s3Key, fileLength, fileModified)
-  }
-  
-  object DbRecord {
-    def apply(original: File)(implicit site: Site): DbRecord =
-      DbRecord(site resolveS3Key original, original.length, original.lastModified, Upload.md5(original).get)
   }
 }

data/src/main/scala/s3/website/Push.scala

@@ -86,55 +86,49 @@ object Push {
     val s3FilesFuture = resolveS3Files()
     val redirectReports: PushReports = redirects.map(S3 uploadRedirect _) map (Right(_))
 
-    val errorsOrReports: Either[ErrorReport, PushReports] = for {
-      diff <- resolveDiff(s3FilesFuture).right
+    val pushReports: Future[PushReports] = for {
+      errorOrUploads: Either[ErrorReport, Seq[Upload]] <- resolveDiff(s3FilesFuture)
     } yield {
-      val newOrChangedReports: PushReports = diff.uploads.map { uploadBatch =>
-        uploadBatch.map(_.right.map(_.map(S3 uploadFile _)))
-      }.map (Await.result(_, 1 day)).foldLeft(Seq(): PushReports) { (memo: PushReports, res: Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]) =>
-        res.fold(
-          error => memo :+ Left(error),
-          (pushResults: Seq[Future[PushErrorOrSuccess]]) => memo ++ (pushResults map (Right(_)))
-        )
-      }
+      val uploadReports: PushReports = errorOrUploads.fold(
+        error => Left(error) :: Nil,
+        uploads => {
+          uploads.map(S3 uploadFile _).map(Right(_))
+        }
+      )
       val deleteReports =
-        Await.result(resolveDeletes(diff, s3FilesFuture, redirects), 1 day).right.map { keysToDelete =>
+        Await.result(resolveDeletes(s3FilesFuture, redirects), 1 day).right.map { keysToDelete =>
           keysToDelete map (S3 delete _)
         }.fold(
           error => Left(error) :: Nil,
           (pushResults: Seq[Future[PushErrorOrSuccess]]) => pushResults map (Right(_))
         )
-      val diffErrorReport: PushReports = Await.result(diff.persistenceError, 1 day).fold(Nil: PushReports)(Left(_) :: Nil)
-      newOrChangedReports ++ deleteReports ++ redirectReports ++ diffErrorReport
+      uploadReports ++ deleteReports ++ redirectReports
     }
-    val errorsOrFinishedPushOps = errorsOrReports.right map awaitForResults
-    val invalidationSucceeded = invalidateCloudFrontItems(errorsOrFinishedPushOps)
+    val finishedPushOps = awaitForResults(Await.result(pushReports, 1 day))
+    val invalidationSucceeded = invalidateCloudFrontItems(finishedPushOps)
     
-    afterPushFinished(errorsOrFinishedPushOps, invalidationSucceeded)
+    afterPushFinished(finishedPushOps, invalidationSucceeded)
   }
   
   def invalidateCloudFrontItems
-    (errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
+    (finishedPushOperations: FinishedPushOperations)
     (implicit config: Config, cloudFrontSettings: CloudFrontSetting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
   Option[InvalidationSucceeded] =
     config.cloudfront_distribution_id.map { distributionId =>
-      val pushSuccessReports = errorsOrFinishedPushOps.fold(
-        errors => Nil,
-        finishedPushOps =>
-          finishedPushOps.map {
-            ops =>
-              for {
-                failedOrSucceededPushes <- ops.right
-                successfulPush <- failedOrSucceededPushes.right
-              } yield successfulPush
-          }.foldLeft(Seq(): Seq[PushSuccessReport]) {
-            (reports, failOrSucc) =>
-              failOrSucc.fold(
-                _ => reports,
-                (pushSuccessReport: PushSuccessReport) => reports :+ pushSuccessReport
-              )
-          }
-      )
+      val pushSuccessReports = 
+        finishedPushOperations.map {
+          ops =>
+            for {
+              failedOrSucceededPushes <- ops.right
+              successfulPush <- failedOrSucceededPushes.right
+            } yield successfulPush
+        }.foldLeft(Seq(): Seq[PushSuccessReport]) {
+          (reports, failOrSucc) =>
+            failOrSucc.fold(
+              _ => reports,
+              (pushSuccessReport: PushSuccessReport) => reports :+ pushSuccessReport
+            )
+        }
       val invalidationResults: Seq[Either[FailedInvalidation, SuccessfulInvalidation]] =
         toInvalidationBatches(pushSuccessReports) map { invalidationBatch =>
           Await.result(
@@ -150,26 +144,25 @@ object Push {
 
   type InvalidationSucceeded = Boolean
 
-  def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])
+  def afterPushFinished(finishedPushOps: FinishedPushOperations, invalidationSucceeded: Option[Boolean])
                        (implicit config: Config, logger: Logger, pushMode: PushMode): ExitCode = {
-    val pushCountsOption = errorsOrFinishedUploads.right.map(resolvePushCounts(_)).right.toOption
-    pushCountsOption.map(pushCountsToString).foreach(pushCounts => logger.info(s"Summary: $pushCounts"))
-    errorsOrFinishedUploads.left foreach (err => logger.fail(s"Encountered an error: ${err.reportMessage}"))
-    val exitCode = errorsOrFinishedUploads.fold(
-      _ => 1,
-      finishedUploads => finishedUploads.foldLeft(0) { (memo, finishedUpload) =>
-        memo + finishedUpload.fold(
-          (error: ErrorReport) => 1,
-          (failedOrSucceededUpload: Either[PushFailureReport, PushSuccessReport]) =>
-            if (failedOrSucceededUpload.isLeft) 1 else 0
-        )
-      } min 1
-    ) max invalidationSucceeded.fold(0)(allInvalidationsSucceeded =>
+    val pushCounts = resolvePushCounts(finishedPushOps)
+    logger.info(s"Summary: ${pushCountsToString(pushCounts)}")
+    val pushOpExitCode = finishedPushOps.foldLeft(0) { (memo, finishedUpload) =>
+      memo + finishedUpload.fold(
+        (error: ErrorReport) => 1,
+        (failedOrSucceededUpload: Either[PushFailureReport, PushSuccessReport]) =>
+          if (failedOrSucceededUpload.isLeft) 1 else 0
+      )
+    } min 1
+    val cloudFrontInvalidationExitCode = invalidationSucceeded.fold(0)(allInvalidationsSucceeded =>
       if (allInvalidationsSucceeded) 0 else 1
     )
 
+    val exitCode = (pushOpExitCode + cloudFrontInvalidationExitCode) min 1
+
     exitCode match {
-      case 0 if !pushMode.dryRun && pushCountsOption.exists(_.thereWasSomethingToPush) =>
+      case 0 if !pushMode.dryRun && pushCounts.thereWasSomethingToPush =>
         logger.info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
       case 1 =>
         logger.fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")

data/src/main/scala/s3/website/S3.scala

@@ -35,7 +35,7 @@ object S3 {
       val report = SuccessfulUpload(
         source.fold(_.s3Key, _.s3Key),
         source.fold(
-          upload => Left(SuccessfulNewOrCreatedDetails(upload.uploadType, upload.uploadFile.get.length(), uploadDuration, upload.reasonForUpload)),
+          upload => Left(SuccessfulNewOrCreatedDetails(upload.uploadType, upload.uploadFile.get.length(), uploadDuration)),
           redirect  => Right(SuccessfulRedirectDetails(redirect.uploadType, redirect.redirectTarget))
         ),
         putObjectRequest
@@ -146,7 +146,7 @@ object S3 {
   }
 
   case class SuccessfulRedirectDetails(uploadType: UploadType, redirectTarget: String)
-  case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long], reasonForUpload: String)
+  case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long])
 
   case class SuccessfulUpload(s3Key: S3Key,
                               details: Either[SuccessfulNewOrCreatedDetails, SuccessfulRedirectDetails],
@@ -167,20 +167,12 @@ object S3 {
           md.getContentEncoding ::
           putObjectRequest.getStorageClass ::
           Nil map (Option(_)) // AWS SDK may return nulls
-        ) :+ uploadSizeForHumans :+ uploadSpeedForHumans :+ uploadReason
+        ) :+ uploadSizeForHumans :+ uploadSpeedForHumans
       detailFragments.collect {
         case Some(detailFragment) => detailFragment
       }.mkString(" | ")
     }
 
-    lazy val uploadReason =
-      details
-        .fold(uploadDetails => Some(uploadDetails.reasonForUpload), _ => None)
-        .collect {
-          case reasonForUpload if logger.verboseOutput =>
-            s"upload reason: $reasonForUpload"
-        }
-
     lazy val uploadSize = details.fold(
       newOrCreatedDetails => Some(newOrCreatedDetails.uploadSize),
       redirectDetails     => None

data/src/main/scala/s3/website/model/push.scala

@@ -50,7 +50,7 @@ case object RedirectFile extends UploadType {
   val pushAction = Redirected
 }
 
-case class Upload(originalFile: File, uploadType: UploadType, reasonForUpload: String)(implicit site: Site) {
+case class Upload(originalFile: File, uploadType: UploadType)(implicit site: Site) {
   lazy val s3Key = site.resolveS3Key(originalFile)
 
   lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -578,65 +578,6 @@ class S3WebsiteSpec extends Specification {
     }
   }
 
-  // Because of the local database, the first and second run are implemented differently.
-  "pushing files for the second time" should {
-    "delete the S3 objects that no longer exist on the local site" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      push
-      setS3Files(S3File("obsolete.txt", ""))
-      push
-      sentDelete must equalTo("obsolete.txt")
-    }
-
-    "delete the local db record for the file if the user deletes the file" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      setLocalFileWithContent(("file.txt", "first run"))
-      push
-      setS3Files(S3File("file.txt", md5Hex("first run")))
-      FileUtils.deleteQuietly(new File(siteDirectory, "file.txt"))
-      push
-      FileUtils.readLines(localDatabase) must beEmpty
-    }
-
-    "push new files to the bucket" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      push
-      setLocalFile("newfile.txt")
-      push
-      sentPutObjectRequest.getKey must equalTo("newfile.txt")
-    }
-
-    "push locally changed files" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      setLocalFileWithContent(("file.txt", "first run"))
-      push
-      setLocalFileWithContent(("file.txt", "second run"))
-      push
-      sentPutObjectRequests.length must equalTo(2)
-    }
-
-    "push locally changed files only once" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      setLocalFileWithContent(("file.txt", "first run"))
-      push
-      setS3Files(S3File("file.txt", md5Hex("first run")))
-      setLocalFileWithContent(("file.txt", "second run"))
-      push
-      sentPutObjectRequests.length must equalTo(2)
-    }
-
-    "detect files that someone else has changed on the S3 bucket" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      setLocalFileWithContent(("file.txt", "first run"))
-      push
-      setOutdatedS3Keys("file.txt")
-      push
-      sentPutObjectRequests.length must equalTo(2)
-    }
-
-    "push locally unchanged files that are missing from S3" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      setLocalFileWithContent(("file.txt", "first run"))
-      push
-      removeAllFilesFromS3()
-      push
-      sentPutObjectRequests.length must equalTo(2) // Even though we use the local db, we should notice that someone else has deleted file.txt
-    }
-  }
-
   "Jekyll site" should {
     "be detected automatically" in new JekyllSite with EmptySite with MockAWS with DefaultRunMode {
       setLocalFile("index.html")
@@ -806,14 +747,13 @@ class S3WebsiteSpec extends Specification {
     implicit final val workingDirectory: File = new File(FileUtils.getTempDirectory, "s3_website_dir" + Random.nextLong())
     val siteDirectory: File // Represents the --site=X option
     val configDirectory: File = workingDirectory // Represents the --config-dir=X option
-    lazy val localDatabase: File = new File(FileUtils.getTempDirectory, "s3_website_local_db_" + sha256Hex(siteDirectory.getPath))
 
     def before {
       workingDirectory :: siteDirectory :: configDirectory :: Nil foreach forceMkdir
     }
 
     def after {
-      (workingDirectory :: siteDirectory :: configDirectory :: localDatabase :: Nil) foreach { dir =>
+      (workingDirectory :: siteDirectory :: configDirectory :: Nil) foreach { dir =>
         if (dir.exists) forceDelete(dir)
       }
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_website
 version: !ruby/object:Gem::Version
-  version: 2.1.6
+  version: 2.1.7
 platform: ruby
 authors:
 - Lauri Lehmijoki