s3_website_monadic 0.0.31 → 0.0.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 240e4d2a14000e5eed029857d69e426cf7439c4f
-  data.tar.gz: ad0d71c36c1f0a8add0cca08f1de9f49467c2019
+  metadata.gz: 87c870c1dae2ea36e452f7df0b0f7565c5cec616
+  data.tar.gz: 4e8324a2207a37a3f991490d187c27d35139c04b
 SHA512:
-  metadata.gz: 17edd14e1510e3b82604e6942a6bca4dc92b3bc34e6189fb4db5f62f2f930b32ea385794f19c1032d53f6d8c8d4600b5bd6cfc8f3ffd08f22f75e05b4b43544f
-  data.tar.gz: b3b818d8d18f40f36cf8b1661785da1fd38b8f4e9afeb78a33a37e39c5cd016bbd0824429330729e65fe01351506c044e51c9c80e372875fd4debebeaf83539f
+  metadata.gz: 5fdcff5e2fad65a9a895bc7669db2690f70c9dd4e7ab54be0962a99a279dda4f88542373eb476e7a6750c2580554888f329aa240ff9d346f26648128adc2a38c
+  data.tar.gz: 0221384872bd4914d9585f5f63074f0b9286600491abdce62f411c5e545d42bd41a484bdf04c686be79350fcb5d71e28d865004f371ef0e32f8caafc04660800

data/changelog.md

@@ -6,6 +6,8 @@ This project uses [Semantic Versioning](http://semver.org).
 
 ### New features
 
+* Faster uploads for extra large sites
+
 * Simulate deployments with `push --dry-run`
 
 * Support CloudFront invalidations when the site contains over 3000 files

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '0.0.31'
+  VERSION = '0.0.32'
 end

data/src/main/scala/s3/website/CloudFront.scala

@@ -1,22 +1,18 @@
 package s3.website
 
-import s3.website.model.{Update, Config}
+import s3.website.model.{FileUpdate, Config}
 import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
-import s3.website.CloudFront.{CloudFrontSetting, SuccessfulInvalidation, FailedInvalidation}
 import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
 import scala.collection.JavaConversions._
 import scala.concurrent.duration._
 import s3.website.S3.{SuccessfulDelete, PushSuccessReport, SuccessfulUpload}
 import com.amazonaws.auth.BasicAWSCredentials
 import java.net.URI
-import Utils._
 import scala.concurrent.{ExecutionContextExecutor, Future}
 
-class CloudFront(implicit cloudFrontSettings: CloudFrontSetting, config: Config, logger: Logger, pushMode: PushMode) {
-  val cloudFront = cloudFrontSettings.cfClient(config)
-
+object CloudFront {
   def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
-                (implicit ec: ExecutionContextExecutor): InvalidationResult =
+                (implicit ec: ExecutionContextExecutor, cloudFrontSettings: CloudFrontSetting, config: Config, logger: Logger, pushMode: PushMode): InvalidationResult =
     Future {
       if (!pushMode.dryRun) cloudFront createInvalidation new CreateInvalidationRequest(distributionId, invalidationBatch)
       val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
@@ -29,7 +25,8 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSetting, config: Config,
     ))
 
   def tooManyInvalidationsRetry(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt)
-                          (implicit ec: ExecutionContextExecutor, logger: Logger): PartialFunction[Throwable, InvalidationResult] = {
+                               (implicit ec: ExecutionContextExecutor, logger: Logger, cloudFrontSettings: CloudFrontSetting, config: Config, pushMode: PushMode):
+  PartialFunction[Throwable, InvalidationResult] = {
     case e: TooManyInvalidationsInProgressException =>
       val duration: Duration = Duration(
         (fibs drop attempt).head min 15, /* CloudFront invalidations complete within 15 minutes */
@@ -52,10 +49,9 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSetting, config: Config,
       basicInfo
   }
 
-  type InvalidationResult = Future[Either[FailedInvalidation, SuccessfulInvalidation]]
-}
+  def cloudFront(implicit config: Config, cloudFrontSettings: CloudFrontSetting) = cloudFrontSettings.cfClient(config)
 
-object CloudFront {
+  type InvalidationResult = Future[Either[FailedInvalidation, SuccessfulInvalidation]]
 
   type CloudFrontClientProvider = (Config) => AmazonCloudFront
 
@@ -121,10 +117,7 @@ object CloudFront {
 
 
   def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
-    case SuccessfulUpload(upload, _, _) => upload.uploadType match {
-      case Update => true
-      case _ => false
-    }
+    case SuccessfulUpload(localFile, _, _) => localFile.left.exists(_.uploadType == FileUpdate)
     case SuccessfulDelete(_) => true
     case _ => false
   }

data/src/main/scala/s3/website/Diff.scala

@@ -2,38 +2,277 @@ package s3.website
 
 import s3.website.model._
 import s3.website.Ruby.rubyRegexMatches
-import s3.website._
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import scala.util.{Failure, Success, Try}
+import java.io.File
+import org.apache.commons.io.FileUtils._
+import org.apache.commons.codec.digest.DigestUtils._
+import scala.io.Source
+import s3.website.Diff.LocalFileDatabase.resolveDiffAgainstLocalDb
+import s3.website.Diff.UploadBatch
+
+case class Diff(
+  unchanged: Future[Either[ErrorReport, Seq[S3Key]]],
+  uploads: Seq[UploadBatch],
+  persistenceError: Future[Option[ErrorReport]]
+)
 
 object Diff {
 
-  def resolveDeletes(localFiles: Seq[LocalFile], s3Files: Seq[S3File], redirects: Seq[Upload with UploadTypeResolved])
-                    (implicit config: Config, logger: Logger): Seq[S3File] = {
-    val keysNotToBeDeleted: Set[String] = (localFiles ++ redirects).map(_.s3Key).toSet
-    s3Files.filterNot { s3File =>
-      val ignoreOnServer = config.ignore_on_server.exists(_.fold(
-        (ignoreRegex: String)        => rubyRegexMatches(s3File.s3Key, ignoreRegex),
-        (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3File.s3Key, _))
-      ))
-      if (ignoreOnServer) logger.debug(s"Ignoring ${s3File.s3Key} on server")
-      keysNotToBeDeleted.exists(_ == s3File.s3Key) || ignoreOnServer
+  type UploadBatch = Future[Either[ErrorReport, Seq[LocalFileFromDisk]]]
+
+  def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
+    if (LocalFileDatabase.hasRecords) resolveDiffAgainstLocalDb(s3FilesFuture)
+    else resolveDiffAgainstGetBucketResponse(s3FilesFuture)
+
+  private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
+    val diffSrc = s3FilesFuture.map { errorOrS3Files =>
+      errorOrS3Files.right.flatMap { s3Files =>
+        Try {
+          val s3KeyIndex = s3Files.map(_.s3Key).toSet
+          val s3Md5Index = s3Files.map(_.md5).toSet
+          val siteFiles = Files.listSiteFiles
+          val nameExistsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
+          val newFiles = siteFiles
+            .filterNot(nameExistsOnS3)
+            .map { f => LocalFileFromDisk(f, uploadType = NewFile)}
+          val changedFiles =
+            siteFiles
+              .filter(nameExistsOnS3)
+              .map(f => LocalFileFromDisk(f, uploadType = FileUpdate))
+              .filterNot(localFile => s3Md5Index contains localFile.md5)
+          val unchangedFiles = {
+            val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
+            siteFiles.filterNot(f => newOrChangedFiles contains f)
+          }
+          val allFiles: Seq[Either[DbRecord, LocalFileFromDisk]] = unchangedFiles.map {
+            f => Left(DbRecord(f))
+          } ++ (changedFiles ++ newFiles).map {
+            Right(_)
+          }
+          LocalFileDatabase persist allFiles
+          allFiles
+        } match {
+          case Success(ok) => Right(ok)
+          case Failure(err) => Left(ErrorReport(err))
+        }
+      }
     }
+    def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFileFromDisk],B]) =
+      diffSrc.map { errorOrDiffSource =>
+        errorOrDiffSource.right map (_ collect pf)
+      }
+    val unchanged = collectResult {
+      case Left(dbRecord) => dbRecord.s3Key
+    }
+    val uploads: UploadBatch = collectResult {
+      case Right(localFile) => localFile
+    }
+    Right(Diff(unchanged, uploads :: Nil, persistenceError = Future(None)))
   }
 
-  def resolveNewFiles(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
-  Stream[Either[ErrorReport, Upload with UploadTypeResolved]] = {
-    val remoteS3KeysIndex = s3Files.map(_.s3Key).toSet
-    localFiles
-      .toStream // Load lazily, because the MD5 computation for the local file requires us to read the whole file
-      .map(resolveUploadSource)
-      .collect {
-      case errorOrUpload if errorOrUpload.right.exists(isNewUpload(remoteS3KeysIndex)) =>
-        for (upload <- errorOrUpload.right) yield upload withUploadType NewFile
+  def resolveDeletes(diff: Diff, s3Files: Future[Either[ErrorReport, Seq[S3File]]], redirects: Seq[Redirect])
+                    (implicit config: Config, logger: Logger, executor: ExecutionContextExecutor): Future[Either[ErrorReport, Seq[S3Key]]] = {
+    val localKeys = for {
+      errorOrUnchanged <- diff.unchanged
+      errorsOrChanges  <- Future.sequence(diff.uploads)
+    } yield {
+      errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
+        for {
+          mem <- memo.right
+          keysToDelete <- errorOrChanges.right
+        } yield {
+          mem ++ keysToDelete.map(_.s3Key)
+        }
+      }
     }
+    s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
+      for {
+        localS3Keys <- errorOrLocalKeys.right
+        remoteS3Keys <- s3Files.right.map(_ map (_.s3Key)).right
+      } yield {
+        val keysToRetain = (localS3Keys ++ (redirects map { _.s3Key })).toSet
+        remoteS3Keys filterNot { s3Key =>
+          val ignoreOnServer = config.ignore_on_server.exists(_.fold(
+            (ignoreRegex: String) => rubyRegexMatches(s3Key, ignoreRegex),
+            (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3Key, _))
+          ))
+          if (ignoreOnServer) logger.debug(s"Ignoring $s3Key on server")
+          (keysToRetain contains s3Key) || ignoreOnServer
+        }
+      }
+    }.tupled
   }
 
-  def isNewUpload(remoteS3KeysIndex: Set[String])(u: Upload) = !remoteS3KeysIndex.exists(_ == u.s3Key)
+  object LocalFileDatabase {
+    def hasRecords(implicit site: Site, logger: Logger) =
+      (for {
+        dbFile <- getOrCreateDbFile
+        databaseIndices <- loadDbFromFile(dbFile)
+      } yield databaseIndices.fullIndex.headOption.isDefined) getOrElse false
+
+    def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
+                                 (implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
+      val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFileFromDisk]]] =
+        (for {
+          dbFile <- getOrCreateDbFile
+          databaseIndices <- loadDbFromFile(dbFile)
+        } yield {
+          val siteFiles = Files.listSiteFiles
+          val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFileFromDisk]]) { (localFiles, file) =>
+            val truncatedKey = TruncatedDbRecord(file)
+            val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
+            if (fileIsUnchanged)
+              localFiles :+ Left(databaseIndices.fullIndex find (_.truncated == truncatedKey) get)
+            else {
+              val uploadType =
+                if (databaseIndices.s3KeyIndex contains truncatedKey.s3Key) FileUpdate
+                else NewFile
+              localFiles :+ Right(LocalFileFromDisk(file, uploadType))
+            }
+          }
+          logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrChangedFiles count (_.isRight)} are new or changed")
+          recordsOrChangedFiles
+        }) match {
+          case Success(ok) => Right(ok)
+          case Failure(err) => Left(ErrorReport(err))
+        }
 
-  def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] =
-    for (upload <- LocalFile.toUpload(localFile).right)
-    yield upload
-}
+      localDiff.right map { localDiffResult =>
+        val unchangedAccordingToLocalDiff = localDiffResult collect {
+          case Left(f) => f
+        }
+
+        val uploadsResolvedByLocalDiff = localDiffResult collect {
+          case Right(f) => f
+        }
+
+        val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFileFromDisk]]] = s3FilesFuture.map { errorOrS3Files =>
+          for (s3Files <- errorOrS3Files.right) yield {
+            val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
+            val localMd5 = unchangedAccordingToLocalDiff.map(_.uploadFileMd5).toSet
+            val changedOnS3 = s3Files.filter { s3File =>
+              (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
+            }
+            logger.debug(s"Detected ${changedOnS3.length} object(s) that have changed on S3 but not on the local site")
+            changedOnS3 map { s3File =>
+              LocalFileFromDisk(site resolveFile s3File, FileUpdate)
+            }
+          }
+        }
+
+        val errorOrDiffAgainstS3 =
+          changesMissedByLocalDiff map { errorOrUploads =>
+            errorOrUploads.right map { uploadsMissedByLocalDiff =>
+              val uploadsS3KeyIndex = uploadsMissedByLocalDiff.map(_.s3Key).toSet
+              val unchangedAccordingToLocalAndS3Diff = unchangedAccordingToLocalDiff.filterNot(uploadsS3KeyIndex contains _.s3Key)
+              (unchangedAccordingToLocalAndS3Diff, uploadsMissedByLocalDiff)
+            }
+          }
+
+        val unchangedFilesFinal = errorOrDiffAgainstS3 map {
+          _ fold (
+            (error: ErrorReport) => Left(error),
+            (syncResult: (Seq[DbRecord], Seq[LocalFileFromDisk])) => Right(syncResult._1)
+          )
+        }
+
+        val changedAccordingToS3Diff = errorOrDiffAgainstS3.map {
+          _ fold (
+            (error: ErrorReport) => Left(error),
+            (syncResult: (Seq[DbRecord], Seq[LocalFileFromDisk])) => Right(syncResult._2)
+          )
+        }
+        val persistenceError: Future[Either[ErrorReport, _]] = for {
+          unchanged <- unchangedFilesFinal
+          changedAccordingToS3 <- changedAccordingToS3Diff
+        } yield
+          for {
+            records1 <- unchanged.right
+            records2 <- changedAccordingToS3.right
+          } yield
+            persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsResolvedByLocalDiff.map(Right(_))) match {
+              case Success(_)   => Unit
+              case Failure(err) => ErrorReport(err)
+            }
+        Diff(
+          unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
+          uploads = Future(Right(uploadsResolvedByLocalDiff)) :: changedAccordingToS3Diff :: Nil,
+          persistenceError = persistenceError map (_.left.toOption)
+        )
+      }
+    }
+
+    private def getOrCreateDbFile(implicit site: Site, logger: Logger) =
+      Try {
+        val dbFile = new File(getTempDirectory, "s3_website_local_db_" + sha256Hex(site.rootDirectory))
+        if (!dbFile.exists()) logger.debug("Creating a new database in " + dbFile.getName)
+        dbFile.createNewFile()
+        dbFile
+      }
+    
+    case class DbIndices(
+      s3KeyIndex:     Set[S3Key],
+      truncatedIndex: Set[TruncatedDbRecord],
+      fullIndex:      Set[DbRecord]
+    ) 
+
+    private def loadDbFromFile(databaseFile: File)(implicit site: Site, logger: Logger): Try[DbIndices] =
+      Try {
+        // record format: "s3Key(file.path)|length(file)|mtime(file)|md5Hex(file.encoded)"
+        val RecordRegex = "(.*?)\\|(\\d+)\\|(\\d+)\\|([a-f0-9]{32})".r
+        val fullIndex = Source
+          .fromFile(databaseFile, "utf-8")
+          .getLines()
+          .toStream
+          .map {
+            case RecordRegex(s3Key, fileLength, fileModified, md5) =>
+              DbRecord(s3Key, fileLength.toLong, fileModified.toLong, md5)
+          }
+          .toSet
+        DbIndices(
+          s3KeyIndex = fullIndex map (_.s3Key), 
+          truncatedIndex = fullIndex map (TruncatedDbRecord(_)),
+          fullIndex
+        )
+      }
+
+    def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFileFromDisk]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFileFromDisk]]] =
+      getOrCreateDbFile flatMap { dbFile =>
+        Try {
+          val dbFileContents = recordsOrChangedFiles.map { recordOrChangedFile =>
+            val record: DbRecord = recordOrChangedFile fold(
+              record => record,
+              changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5)
+              )
+            record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
+          } mkString "\n"
+
+          write(dbFile, dbFileContents)
+          recordsOrChangedFiles
+        }
+      }
+  }
+
+  case class TruncatedDbRecord(s3Key: String, fileLength: Long, fileModified: Long)
+
+  object TruncatedDbRecord {
+    def apply(dbRecord: DbRecord): TruncatedDbRecord = TruncatedDbRecord(dbRecord.s3Key, dbRecord.fileLength, dbRecord.fileModified)
+    
+    def apply(file: File)(implicit site: Site): TruncatedDbRecord = TruncatedDbRecord(site resolveS3Key file, file.length, file.lastModified)
+  }
+
+  /**
+   * @param uploadFileMd5 if the file is gzipped, this checksum should be calculated on the gzipped file, not the original file
+   */
+  case class DbRecord(s3Key: String, fileLength: Long, fileModified: Long, uploadFileMd5: MD5) {
+    lazy val truncated = TruncatedDbRecord(s3Key, fileLength, fileModified)
+  }
+  
+  object DbRecord {
+    def apply(original: File)(implicit site: Site): DbRecord =
+      DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFileFromDisk.md5(original))
+  }
+}

data/src/main/scala/s3/website/Logger.scala

@@ -0,0 +1,61 @@
+package s3.website
+
+class Logger(val verboseOutput: Boolean, logMessage: (String) => Unit = println) {
+  def debug(msg: String) = if (verboseOutput) log(Debug, msg)
+  def info(msg: String) = log(Info, msg)
+  def fail(msg: String) = log(Failure, msg)
+
+  def info(report: SuccessReport) = log(Success, report.reportMessage)
+  def info(report: FailureReport) = fail(report.reportMessage)
+
+  def pending(msg: String) = log(Wait, msg)
+
+  private def log(logType: LogType, msgRaw: String) {
+    val msg = msgRaw.replaceAll("\\n", "\n       ") // Indent new lines, so that they arrange nicely with other log lines
+    logMessage(s"[$logType] $msg")
+  }
+
+  sealed trait LogType {
+    val prefix: String
+    override def toString = prefix
+  }
+  case object Debug extends LogType {
+    val prefix = "debg".cyan
+  }
+  case object Info extends LogType {
+    val prefix = "info".blue
+  }
+  case object Success extends LogType {
+    val prefix = "succ".green
+  }
+  case object Failure extends LogType {
+    val prefix = "fail".red
+  }
+  case object Wait extends LogType {
+    val prefix = "wait".yellow
+  }
+
+  /**
+   * Idea copied from https://github.com/ktoso/scala-rainbow.
+   */
+  implicit class RainbowString(val s: String) {
+    import Console._
+
+    /** Colorize the given string foreground to ANSI black */
+    def black = BLACK + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def red = RED + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def green = GREEN + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def yellow = YELLOW + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def blue = BLUE + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def magenta = MAGENTA + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def cyan = CYAN + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def white = WHITE + s + RESET
+  }
+}