s3_website 1.8.2 → 2.0.0

data/build.sbt

@@ -0,0 +1,35 @@
+import AssemblyKeys._
+
+name := "s3_website"
+
+version := "0.0.1"
+
+scalaVersion := "2.11.1"
+
+scalacOptions += "-feature"
+
+scalacOptions += "-language:implicitConversions"
+
+scalacOptions += "-language:postfixOps"
+
+libraryDependencies += "org.yaml" % "snakeyaml" % "1.13"
+
+libraryDependencies += "org.jruby" % "jruby" % "1.7.11"
+
+libraryDependencies += "com.amazonaws" % "aws-java-sdk" % "1.7.7"
+
+libraryDependencies += "commons-codec" % "commons-codec" % "1.9"
+
+libraryDependencies += "commons-io" % "commons-io" % "2.4"
+
+libraryDependencies += "org.apache.tika" % "tika-core" % "1.4"
+
+libraryDependencies += "com.lexicalscope.jewelcli" % "jewelcli" % "0.8.9"
+
+libraryDependencies += "org.specs2" %% "specs2" % "2.3.11" % "test"
+
+resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"
+
+jarName in assembly := "s3_website.jar"
+
+test in assembly := {}

data/changelog.md

@@ -2,9 +2,60 @@
 
 This project uses [Semantic Versioning](http://semver.org).
 
-## 1.8.2
+## 2.0.0
 
-* do not match files like denver.html with a bad regex
+### New features
+
+* Faster uploads for extra large sites
+
+   Use a local database for calculating diffs. This removes the need to read all
+   the files of the website, when you call the `s3_website push` command.
+
+   Use proper multithreading with JVM threads.
+
+* Simulate deployments with `push --dry-run`
+
+* Support CloudFront invalidations when the site contains over 3000 files
+
+* Display transferred bytes
+
+* Display upload speed
+
+* `push --verbose` switch
+
+### Bug fixes
+
+* Fault tolerance – do not crash if one of the uploads fails
+
+   Before, the push command crashed if something unexpected happened. From now
+   on, s3_website will run all the operations it can, and report errors in the
+   end.
+
+### Java is now required
+
+* The `push` command is now written in Scala
+
+   This means that you need Java 1.6 or above to run the command `s3_website
+   push`.
+
+### Removed features
+
+* `extensionless_mime_type`
+
+    s3_website now relies on Apache Tika to infer the mime type.
+
+* the `--headless` switch on the command-line
+
+    s3_website always deletes the files that are on the S3 bucket but not on the local file system.
+    Use the settings `ignore_on_server` and `exclude_from_upload` to control the retained files.
+
+* You can no longer use this gem as a Ruby library. You can migrate by calling
+  the `s3_website push --site=x --config-dir=y` system command from your Ruby code.
+
+* `gzip_zopfli: true`
+
+    At the time of writing, there does not exist a stable zopfli implementation
+    for Java.
 
 ## 1.8.1
 

data/lib/s3_website.rb

@@ -1,38 +1,7 @@
 require 'rubygems'
-require 'yaml'
-require 'erubis'
-require 'aws-sdk'
-require 'simple-cloudfront-invalidator'
-require 'filey-diff'
-require 'mime/types'
 require 'thor'
 require 'dotenv'
 
 Dotenv.load
 
-module S3Website
-  DEFAULT_GZIP_EXTENSIONS = %w(.html .css .js .svg .txt)
-end
-
-%w{
-  errors
-  upload
-  uploader
-  tasks
-  config_loader
-  retry
-  keyboard
-  diff_helper
-  gzip_helper
-  endpoint
-  parallelism
-  jekyll
-  nanoc
-  paths
-}.each do |file|
-  require File.dirname(__FILE__) + "/s3_website/#{file}"
-end
-
-%w{invalidator}.each do |file|
-  require File.dirname(__FILE__) + "/cloudfront/#{file}"
-end
+require File.dirname(__FILE__) + "/s3_website/version"

data/lib/s3_website/version.rb

@@ -0,0 +1,3 @@
+module S3Website
+  VERSION = '2.0.0'
+end

data/project/assembly.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")

data/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.github.mpeltonen" % "sbt-idea" % "1.6.0")

data/release

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'octokit'
+require_relative 'lib/s3_website/version'
+
+version = S3Website::VERSION
+unless ENV['S3_WEBSITE_RELEASE_KEY']
+  puts "Set the S3_WEBSITE_RELEASE_KEY environmental variable. It should be your personal GitHub access token."
+  exit 1
+end
+
+def run(cmd)
+  unless system(cmd)
+    exit 1
+  end
+end
+
+puts "Building s3_website.jar..."
+run "./sbt clean"
+run "./sbt assembly"
+
+tag_name = "v#{version}"
+run "git tag #{tag_name} && git push origin #{tag_name}"
+run "git push"
+
+run "rake build"
+
+client = Octokit::Client.new(:login => 'laurilehmijoki', :password => ENV['S3_WEBSITE_RELEASE_KEY'])
+release = client.create_release(
+  'laurilehmijoki/s3_website', 
+  tag_name,
+  :body => "See https://github.com/laurilehmijoki/s3_website/blob/master/changelog.md##{version.gsub('.', '')}"
+)
+puts "Uploading s3_website.jar..."
+client.upload_asset(release[:url], 'target/scala-2.11/s3_website.jar')
+run "gem push pkg/s3_website_monadic-#{version}.gem"

data/resources/configuration_file_template.yml

@@ -52,5 +52,3 @@ s3_bucket: your.blog.bucket.com
 #       host_name: blog.example.com
 #       replace_key_prefix_with: some_new_path/
 #       http_redirect_code: 301
-
-# extensionless_mime_type: text/html

data/s3_website.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
+require File.join([File.dirname(__FILE__),'lib','s3_website','version.rb'])
 
 Gem::Specification.new do |s|
   s.name        = "s3_website"
-  s.version     = "1.8.2"
+  s.version     = S3Website::VERSION
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Lauri Lehmijoki"]
   s.email       = ["lauri.lehmijoki@iki.fi"]
@@ -15,28 +15,18 @@ Gem::Specification.new do |s|
   }
   s.license     = 'MIT'
 
-  s.default_executable = %q{s3_website}
+  s.default_executable = %q{s3_website_monadic}
 
-  s.add_dependency 'aws-sdk', '~> 1'
-  s.add_dependency 'filey-diff', '~> 2.0.0'
-  s.add_dependency 'simple-cloudfront-invalidator', '~> 1'
-  s.add_dependency 'erubis', '~> 2.7.0'
-  s.add_dependency 'mime-types', '~> 1'
   s.add_dependency 'thor', '= 0.18.1'
   s.add_dependency 'configure-s3-website', '= 1.5.5'
-  s.add_dependency 'zopfli', '~> 0.0.3'
+  s.add_dependency 'colored', '1.2'
   s.add_dependency 'dotenv', '~> 0.11.1'
 
-  s.add_development_dependency 'rspec', '2.14.0'
-  s.add_development_dependency 'rspec-expectations', '2.14.4'
-  s.add_development_dependency 'cucumber', '1.3.10'
-  s.add_development_dependency 'aruba', '0.5.3'
   s.add_development_dependency 'rake', '10.1.1'
-  s.add_development_dependency 'vcr', '2.8.0'
-  s.add_development_dependency 'webmock', '1.16.1'
+  s.add_development_dependency 'octokit', '3.1.0'
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.files         = `git ls-files`.split("\n").reject { |f| f.match('sbt-launch.jar') }
+  s.test_files    = `git ls-files -- src/test/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 end

data/sbt

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+SBT_OPTS="-Xms512M -Xmx1536M -Xss1M -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=256M"
+java $SBT_OPTS -jar `dirname $0`/project/sbt-launch.jar "$@"

data/src/main/java/s3/website/ByteHelper.java

@@ -0,0 +1,14 @@
+package s3.website;
+
+public class ByteHelper {
+
+    // Adapted from http://stackoverflow.com/a/3758880/219947
+    public static String humanReadableByteCount(long bytes) {
+        boolean si = true;
+        int unit = si ? 1000 : 1024;
+        if (bytes < unit) return bytes + " B";
+        int exp = (int) (Math.log(bytes) / Math.log(unit));
+        String pre = (si ? "kMGTPE" : "KMGTPE").charAt(exp-1) + (si ? "" : "i");
+        return String.format("%.1f %sB", bytes / Math.pow(unit, exp), pre);
+    }
+}

data/src/main/scala/s3/website/CloudFront.scala

@@ -0,0 +1,129 @@
+package s3.website
+
+import s3.website.model.{FileUpdate, Config}
+import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
+import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
+import scala.collection.JavaConversions._
+import scala.concurrent.duration._
+import s3.website.S3.{SuccessfulDelete, PushSuccessReport, SuccessfulUpload}
+import com.amazonaws.auth.BasicAWSCredentials
+import java.net.URI
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import s3.website.model.Config.awsCredentials
+
+object CloudFront {
+  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
+                (implicit ec: ExecutionContextExecutor, cloudFrontSettings: CloudFrontSetting, config: Config, logger: Logger, pushMode: PushMode): InvalidationResult =
+    Future {
+      if (!pushMode.dryRun) cloudFront createInvalidation new CreateInvalidationRequest(distributionId, invalidationBatch)
+      val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
+      logger.debug(invalidationBatch.getPaths.getItems.map(item => s"${Invalidated.renderVerb} $item") mkString "\n")
+      logger.info(result)
+      Right(result)
+    } recoverWith (tooManyInvalidationsRetry(invalidationBatch, distributionId, attempt) orElse retry(attempt)(
+      createFailureReport = error => FailedInvalidation(error),
+      retryAction = nextAttempt => invalidate(invalidationBatch, distributionId, nextAttempt)
+    ))
+
+  def tooManyInvalidationsRetry(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt)
+                               (implicit ec: ExecutionContextExecutor, logger: Logger, cloudFrontSettings: CloudFrontSetting, config: Config, pushMode: PushMode):
+  PartialFunction[Throwable, InvalidationResult] = {
+    case e: TooManyInvalidationsInProgressException =>
+      val duration: Duration = Duration(
+        (fibs drop attempt).head min 15, /* CloudFront invalidations complete within 15 minutes */
+        cloudFrontSettings.retryTimeUnit
+      )
+      logger.pending(maxInvalidationsExceededInfo(duration, attempt))
+      Thread.sleep(duration.toMillis)
+      invalidate(invalidationBatch, distributionId, attempt + 1)
+  }
+
+  def maxInvalidationsExceededInfo(sleepDuration: Duration, attempt: Int) = {
+    val basicInfo = s"The maximum amount of CloudFront invalidations has exceeded. Trying again in $sleepDuration, please wait."
+    val extendedInfo =
+      s"""|$basicInfo
+          |For more information, see http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits"""
+        .stripMargin
+    if (attempt == 1)
+      extendedInfo
+    else
+      basicInfo
+  }
+
+  def cloudFront(implicit config: Config, cloudFrontSettings: CloudFrontSetting) = cloudFrontSettings.cfClient(config)
+
+  type InvalidationResult = Future[Either[FailedInvalidation, SuccessfulInvalidation]]
+
+  type CloudFrontClientProvider = (Config) => AmazonCloudFront
+
+  case class SuccessfulInvalidation(invalidatedItemsCount: Int)(implicit pushMode: PushMode) extends SuccessReport {
+    def reportMessage = s"${Invalidated.renderVerb} ${invalidatedItemsCount ofType "item"} on CloudFront"
+  }
+
+  case class FailedInvalidation(error: Throwable) extends FailureReport {
+    def reportMessage = s"Failed to invalidate the CloudFront distribution (${error.getMessage})"
+  }
+
+  def awsCloudFrontClient(config: Config) = new AmazonCloudFrontClient(awsCredentials(config))
+  
+  def toInvalidationBatches(pushSuccessReports: Seq[PushSuccessReport])(implicit config: Config): Seq[InvalidationBatch] = {
+    val invalidationPaths: Seq[String] = {
+      def withDefaultPathIfNeeded(paths: Seq[String]) = {
+        // This is how we support the Default Root Object @ CloudFront (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DefaultRootObject.html)
+        // We do this more accurately by fetching the distribution config (http://docs.aws.amazon.com/AmazonCloudFront/latest/APIReference/GetConfig.html)
+        // and reading the Default Root Object from there.
+        val containsPotentialDefaultRootObject = paths
+          .exists(
+            _
+              .replaceFirst("^/", "") // S3 keys do not begin with a slash
+              .contains("/") == false // See if the S3 key is a top-level key (i.e., it is not within a directory)
+          )
+        if (containsPotentialDefaultRootObject) paths :+ "/" else paths
+      }
+      val paths = pushSuccessReports
+        .filter(needsInvalidation) // Assume that redirect objects are never cached.
+        .map(toInvalidationPath)
+        .map (applyInvalidateRootSetting)
+      withDefaultPathIfNeeded(paths)
+    }
+
+    invalidationPaths
+      .grouped(1000) // CloudFront supports max 1000 invalidations in one request (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits)
+      .map { batchKeys =>
+        new InvalidationBatch() withPaths
+          (new Paths() withItems batchKeys withQuantity batchKeys.size) withCallerReference
+          s"s3_website gem ${System.currentTimeMillis()}"
+      }
+      .toSeq
+  }
+
+  def applyInvalidateRootSetting(path: String)(implicit config: Config) =
+    if (config.cloudfront_invalidate_root.exists(_ == true))
+      path.replaceFirst("/index.html$", "/")
+    else
+      path
+
+  def toInvalidationPath(report: PushSuccessReport) = {
+    def encodeUnsafeChars(path: String) =
+      new URI(
+        "http",
+        "cloudfront", // We want to use the encoder in the URI class. These must be passed in.
+        "/" + report.s3Key,  // CloudFront keys have the slash in front
+        path
+      ).toURL.getPath // The URL class encodes the unsafe characters
+    val invalidationPath = "/" + report.s3Key  // CloudFront keys have the slash in front
+    encodeUnsafeChars(invalidationPath)
+  }
+
+
+  def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
+    case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) == FileUpdate
+    case SuccessfulDelete(_) => true
+    case _ => false
+  }
+
+  case class CloudFrontSetting(
+    cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
+    retryTimeUnit: TimeUnit = MINUTES
+  ) extends RetrySetting
+}

data/src/main/scala/s3/website/Diff.scala

@@ -0,0 +1,280 @@
+package s3.website
+
+import s3.website.model._
+import s3.website.Ruby.rubyRegexMatches
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import scala.util.{Failure, Success, Try}
+import java.io.File
+import org.apache.commons.io.FileUtils._
+import org.apache.commons.codec.digest.DigestUtils._
+import scala.io.Source
+import s3.website.Diff.LocalFileDatabase.resolveDiffAgainstLocalDb
+import s3.website.Diff.UploadBatch
+
+case class Diff(
+  unchanged: Future[Either[ErrorReport, Seq[S3Key]]],
+  uploads: Seq[UploadBatch],
+  persistenceError: Future[Option[ErrorReport]]
+)
+
+object Diff {
+
+  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFile]]]
+
+  def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
+    if (LocalFileDatabase.hasRecords) resolveDiffAgainstLocalDb(s3FilesFuture)
+    else resolveDiffAgainstGetBucketResponse(s3FilesFuture)
+
+  private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
+    val diffAgainstS3 = s3FilesFuture.map { errorOrS3Files =>
+      errorOrS3Files.right.flatMap { s3Files =>
+        Try {
+          val s3KeyIndex = s3Files.map(_.s3Key).toSet
+          val s3Md5Index = s3Files.map(_.md5).toSet
+          val siteFiles = Files.listSiteFiles
+          val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
+          val isChangedOnS3 = (localFile: LocalFile) => !(s3Md5Index contains localFile.md5.get)
+          val newFiles = siteFiles collect {
+            case file if !existsOnS3(file) => LocalFile(file, NewFile)
+          }
+          val changedFiles = siteFiles collect {
+            case file if existsOnS3(file) => LocalFile(file, FileUpdate)
+          } filter isChangedOnS3
+          val unchangedFiles = {
+            val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
+            siteFiles.filterNot(f => newOrChangedFiles contains f)
+          }
+          val allFiles: Seq[Either[DbRecord, LocalFile]] = unchangedFiles.map {
+            f => Left(DbRecord(f))
+          } ++ (changedFiles ++ newFiles).map {
+            Right(_)
+          }
+          LocalFileDatabase persist allFiles
+          allFiles
+        } match {
+          case Success(ok) => Right(ok)
+          case Failure(err) => Left(ErrorReport(err))
+        }
+      }
+    }
+    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFile],B]) =
+      diffAgainstS3.map { errorOrDiffSource =>
+        errorOrDiffSource.right map (_ collect pf)
+      }
+    val unchanged = collectResult {
+      case Left(dbRecord) => dbRecord.s3Key
+    }
+    val uploads: UploadBatch = collectResult {
+      case Right(localFile) => localFile
+    }
+    Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
+  }
+
+  def resolveDeletes(diff: Diff, s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
+                    (implicit config: Config, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = {
+    val localKeys = for {
+      errorOrUnchanged <- diff.unchanged
+      errorsOrChanges  <- Future.sequence(diff.uploads)
+    } yield
+      errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
+        for {
+          mem <- memo.right
+          keysToDelete <- errorOrChanges.right
+        } yield mem ++ keysToDelete.map(_.s3Key)
+      }
+
+    s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
+      for {
+        localS3Keys <- errorOrLocalKeys.right
+        remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
+      } yield {
+        val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet
+        remoteS3Keys filterNot { s3Key =>
+          val ignoreOnServer = config.ignore_on_server.exists(_.fold(
+            (ignoreRegex: String) => rubyRegexMatches(s3Key, ignoreRegex),
+            (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3Key, _))
+          ))
+          if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server")
+          (keysToRetain contains s3Key) || ignoreOnServer
+        }
+      }
+    }.tupled
+  }
+
+  object LocalFileDatabase {
+    def hasRecords(implicit site: Site, logger: Logger) =
+      (for {
+        dbFile <- getOrCreateDbFile
+        databaseIndices <- loadDbFromFile(dbFile)
+      } yield databaseIndices.fullIndex.headOption.isDefined) getOrElse false
+
+    def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
+      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFile]]] =
+        (for {
+          dbFile <- getOrCreateDbFile
+          databaseIndices <- loadDbFromFile(dbFile)
+        } yield {
+          val siteFiles = Files.listSiteFiles
+          val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFile]]) { (localFiles, file) =>
+            val truncatedKey = TruncatedDbRecord(file)
+            val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
+            if (fileIsUnchanged)
+              localFiles :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
+            else {
+              val uploadType =
+                if (databaseIndices.s3KeyIndex contains truncatedKey.s3Key) FileUpdate
+                else NewFile
+              localFiles :+ Right(LocalFile(file, uploadType))
+            }
+          }
+          logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrChangedFiles count (_.isRight)} are new or changed")
+          recordsOrChangedFiles
+        }) match {
+          case Success(ok) => Right(ok)
+          case Failure(err) => Left(ErrorReport(err))
+        }
+
+      localDiff.right map { localDiffResult =>
+        val unchangedAccordingToLocalDiff = localDiffResult collect {
+          case Left(f) => f
+        }
+
+        val uploadsAccordingToLocalDiff = localDiffResult collect {
+          case Right(f) => f
+        }
+
+        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFile]]] = s3FilesFuture.map { errorOrS3Files =>
+          for (s3Files <- errorOrS3Files.right) yield {
+            val remoteS3Keys = s3Files.map(_.s3Key).toSet
+            val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
+            val localMd5 = unchangedAccordingToLocalDiff.map(_.uploadFileMd5).toSet
+            def isChangedOnS3(s3File: S3File) = (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
+            val changedOnS3 = s3Files collect {
+              case s3File if isChangedOnS3(s3File) =>
+                LocalFile(site resolveFile s3File, FileUpdate)
+            }
+            val missingFromS3 = localS3Keys collect {
+              case localS3Key if !(remoteS3Keys contains localS3Key) =>
+                LocalFile(site resolveFile localS3Key, NewFile)
+
+            }
+            changedOnS3 ++ missingFromS3
+          }
+        }
+
+        val errorOrDiffAgainstS3 =
+          changesMissedByLocalDiff map { errorOrUploads =>
+            errorOrUploads.right map { uploadsMissedByLocalDiff =>
+              val uploadsS3KeyIndex = uploadsMissedByLocalDiff.map(_.s3Key).toSet
+              val unchangedAccordingToLocalAndS3Diff = unchangedAccordingToLocalDiff.filterNot(uploadsS3KeyIndex contains _.s3Key)
+              (unchangedAccordingToLocalAndS3Diff, uploadsMissedByLocalDiff)
+            }
+          }
+
+        val unchangedFilesFinal = errorOrDiffAgainstS3 map {
+          _ fold (
+            (error: ErrorReport) => Left(error),
+            (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._1)
+          )
+        }
+
+        val changedAccordingToS3Diff = errorOrDiffAgainstS3.map {
+          _ fold (
+            (error: ErrorReport) => Left(error),
+            (syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._2)
+          )
+        }
+        val persistenceError: Future[Either[ErrorReport, _]] = for {
+          unchanged <- unchangedFilesFinal
+          changedAccordingToS3 <- changedAccordingToS3Diff
+        } yield
+          for {
+            records1 <- unchanged.right
+            records2 <- changedAccordingToS3.right
+          } yield
+            persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsAccordingToLocalDiff.map(Right(_))) match {
+              case Success(_)   => Unit
+              case Failure(err) => ErrorReport(err)
+            }
+        Diff(
+          unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
+          uploads = Future(Right(uploadsAccordingToLocalDiff)) :: changedAccordingToS3Diff :: Nil,
+          persistenceError = persistenceError map (_.left.toOption)
+        )
+      }
+    }
+
+    private def getOrCreateDbFile(implicit site: Site, logger: Logger) =
+      Try {
+        val dbFile = new File(getTempDirectory, "s3_website_local_db_" + sha256Hex(site.rootDirectory))
+        if (!dbFile.exists()) logger.debug("Creating a new database in " + dbFile.getName)
+        dbFile.createNewFile()
+        dbFile
+      }
+    
+    case class DbIndices(
+      s3KeyIndex:     Set[S3Key],
+      truncatedIndex: Set[TruncatedDbRecord],
+      fullIndex:      Set[DbRecord]
+    ) 
+
+    private def loadDbFromFile(databaseFile: File)(implicit site: Site, logger: Logger): Try[DbIndices] =
+      Try {
+        // record format: "s3Key(file.path)|length(file)|mtime(file)|md5Hex(file.encoded)"
+        val RecordRegex = "(.*?)\\|(\\d+)\\|(\\d+)\\|([a-f0-9]{32})".r
+        val fullIndex = Source
+          .fromFile(databaseFile, "utf-8")
+          .getLines()
+          .toStream
+          .map {
+            case RecordRegex(s3Key, fileLength, fileModified, md5) =>
+              DbRecord(s3Key, fileLength.toLong, fileModified.toLong, md5)
+          }
+          .toSet
+        DbIndices(
+          s3KeyIndex = fullIndex map (_.s3Key), 
+          truncatedIndex = fullIndex map (TruncatedDbRecord(_)),
+          fullIndex
+        )
+      }
+
+    def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFile]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFile]]] =
+      getOrCreateDbFile flatMap { dbFile =>
+        Try {
+          val dbFileContents = recordsOrChangedFiles.map { recordOrChangedFile =>
+            val record: DbRecord = recordOrChangedFile fold(
+              record => record,
+              changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5.get)
+              )
+            record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
+          } mkString "\n"
+
+          write(dbFile, dbFileContents)
+          recordsOrChangedFiles
+        }
+      }
+  }
+
+  case class TruncatedDbRecord(s3Key: String, fileLength: Long, fileModified: Long)
+
+  object TruncatedDbRecord {
+    def apply(dbRecord: DbRecord): TruncatedDbRecord = TruncatedDbRecord(dbRecord.s3Key, dbRecord.fileLength, dbRecord.fileModified)
+    
+    def apply(file: File)(implicit site: Site): TruncatedDbRecord = TruncatedDbRecord(site resolveS3Key file, file.length, file.lastModified)
+  }
+
+  /**
+   * @param uploadFileMd5 if the file is gzipped, this checksum should be calculated on the gzipped file, not the original file
+   */
+  case class DbRecord(s3Key: String, fileLength: Long, fileModified: Long, uploadFileMd5: MD5) {
+    lazy val truncated = TruncatedDbRecord(s3Key, fileLength, fileModified)
+  }
+  
+  object DbRecord {
+    def apply(original: File)(implicit site: Site): DbRecord =
+      DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFile.md5(original).get)
+  }
+}