s3_website_revived 4.0.0

data/resources/configuration_file_template.yml

@@ -0,0 +1,67 @@
+# You can remove the first two lines to have credentials read from
+# the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY or
+# ~/.aws/credentials.
+s3_id: YOUR_AWS_S3_ACCESS_KEY_ID
+s3_secret: YOUR_AWS_S3_SECRET_ACCESS_KEY
+s3_bucket: your.blog.bucket.com
+
+# set session_token if using temporary credentials with a session token (eg: when assuming a role)
+# session_token: YOUR_AWS_S3_SESSION_TOKEN
+
+# Below are examples of all the available configurations.
+# See README for more detailed info on each of them.
+
+# site: path-to-your-website
+
+# index_document: index.html
+# error_document: error.html
+
+# max_age:
+#   "assets/*": 6000
+#   "*": 300
+
+# gzip:
+#   - .html
+#   - .css
+#   - .md
+# gzip_zopfli: true
+
+# See http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region for valid endpoints
+# s3_endpoint: ap-northeast-1
+
+# ignore_on_server: that_folder_of_stuff_i_dont_keep_locally
+
+# exclude_from_upload:
+#   - those_folders_of_stuff
+#   - i_wouldnt_want_to_upload
+
+# s3_reduced_redundancy: true
+
+# cloudfront_distribution_id: your-dist-id
+
+# cloudfront_distribution_config:
+#   default_cache_behavior:
+#     min_ttl: <%= 60 * 60 * 24 %>
+#   aliases:
+#     quantity: 1
+#     items:
+#       - your.website.com
+
+# cloudfront_invalidate_root: true
+
+cloudfront_wildcard_invalidation: true
+
+# concurrency_level: 5
+
+# redirects:
+#   index.php: /
+#   about.php: about.html
+#   music-files/promo.mp4: http://www.youtube.com/watch?v=dQw4w9WgXcQ
+
+# routing_rules:
+#   - condition:
+#       key_prefix_equals: blog/some_path
+#     redirect:
+#       host_name: blog.example.com
+#       replace_key_prefix_with: some_new_path/
+#       http_redirect_code: 301

data/resources/s3_website.jar.md5

@@ -0,0 +1 @@
+02bbb61fa5042bf4a79b24b660aab227  s3_website-4.0.0.jar

data/s3_website.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+require File.join([File.dirname(__FILE__),'lib','s3_website','version.rb'])
+
+Gem::Specification.new do |s|
+  s.name        = "s3_website_revived"
+  s.version     = S3Website::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ivo Anjo", "Lauri Lehmijoki"]
+  s.email       = ["ivo@ivoanjo.me"]
+  s.homepage    = "https://github.com/ivoanjo/s3_website_revived"
+  s.summary     = %q{Revived fork of s3_website - Manage your S3 website}
+  s.description = %q{
+    Revived fork of s3_website. Sync website files, set redirects, use HTTP performance optimisations, deliver via
+    CloudFront.
+  }
+  s.license     = 'MIT'
+
+  s.add_dependency 'thor', '~> 0.18'
+  s.add_dependency 'configure-s3-website', '= 2.3.0'
+  s.add_dependency 'colored', '1.2'
+  s.add_dependency 'dotenv', '~> 1.0'
+
+  s.add_development_dependency 'rake', '10.1.1'
+  s.add_development_dependency 'octokit', '3.1.0'
+  s.add_development_dependency 'mime-types'
+
+  s.files         = `git ls-files`
+                      .split("\n")
+                      .reject { |f| f.match('sbt-launch.jar') } # Reject the SBT jar, as it is a big file
+                      .push('resources/s3_website.jar.md5') # Include the checksum file in the gem
+  s.test_files    = `git ls-files -- src/test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/sbt

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+java -jar `dirname $0`/project/sbt-launch.jar "$@"

data/src/main/resources/log4j.properties

@@ -0,0 +1,6 @@
+log4j.rootLogger=WARN, A1
+log4j.appender.A1=org.apache.log4j.ConsoleAppender
+log4j.appender.A1.layout=org.apache.log4j.PatternLayout
+log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c -  %m%n
+# Or you can explicitly enable WARN and ERROR messages for  the AWS Java clients
+log4j.logger.com.amazonaws=WARN

data/src/main/scala/s3/website/ByteHelper.scala

@@ -0,0 +1,18 @@
+package s3.website
+
+object ByteHelper {
+
+  // Adapted from http://stackoverflow.com/a/3758880/219947
+  def humanReadableByteCount(bytes: Long): String = {
+    val si: Boolean = true
+    val unit: Int = if (si) 1000 else 1024
+    if (bytes < unit) {
+      bytes + " B"
+    } else {
+      val exp: Int = (Math.log(bytes) / Math.log(unit)).asInstanceOf[Int]
+      val pre: String = (if (si) "kMGTPE" else "KMGTPE").charAt(exp - 1) + (if (si) "" else "i")
+      val formatArgs = (bytes / Math.pow(unit, exp)).asInstanceOf[AnyRef] :: pre :: Nil
+      String.format("%.1f %sB", formatArgs.toArray:_*)
+    }
+  }
+}

data/src/main/scala/s3/website/CloudFront.scala

@@ -0,0 +1,144 @@
+package s3.website
+
+import s3.website.ErrorReport._
+import s3.website.model.{FileUpdate, Config}
+import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
+import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
+import scala.collection.JavaConversions._
+import scala.concurrent.duration._
+import s3.website.S3.{SuccessfulDelete, PushSuccessReport, SuccessfulUpload}
+import com.amazonaws.auth.BasicAWSCredentials
+import java.net.{URLEncoder, URI}
+import scala.concurrent.{ExecutionContextExecutor, Future}
+import s3.website.model.Config.awsCredentials
+
+object CloudFront {
+  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
+                (implicit ec: ExecutionContextExecutor, cloudFrontSettings: CloudFrontSetting, config: Config, logger: Logger, pushOptions: PushOptions): InvalidationResult =
+    Future {
+      if (!pushOptions.dryRun) cloudFront createInvalidation new CreateInvalidationRequest(distributionId, invalidationBatch)
+      val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
+      logger.debug(invalidationBatch.getPaths.getItems.map(item => s"${Invalidated.renderVerb} $item") mkString "\n")
+      logger.info(result)
+      Right(result)
+    } recoverWith (tooManyInvalidationsRetry(invalidationBatch, distributionId, attempt) orElse retry(attempt)(
+      createFailureReport = error => FailedInvalidation(error),
+      retryAction = nextAttempt => invalidate(invalidationBatch, distributionId, nextAttempt)
+    ))
+
+  def tooManyInvalidationsRetry(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt)
+                               (implicit ec: ExecutionContextExecutor, logger: Logger, cloudFrontSettings: CloudFrontSetting, config: Config, pushOptions: PushOptions):
+  PartialFunction[Throwable, InvalidationResult] = {
+    case e: TooManyInvalidationsInProgressException =>
+      val duration: Duration = Duration(
+        (fibs drop attempt).head min 15, /* CloudFront invalidations complete within 15 minutes */
+        cloudFrontSettings.retryTimeUnit
+      )
+      logger.pending(maxInvalidationsExceededInfo(duration, attempt))
+      Thread.sleep(duration.toMillis)
+      invalidate(invalidationBatch, distributionId, attempt + 1)
+  }
+
+  def maxInvalidationsExceededInfo(sleepDuration: Duration, attempt: Int) = {
+    val basicInfo = s"The maximum amount of CloudFront invalidations has exceeded. Trying again in $sleepDuration, please wait."
+    val extendedInfo =
+      s"""|$basicInfo
+          |For more information, see http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits"""
+        .stripMargin
+    if (attempt == 1)
+      extendedInfo
+    else
+      basicInfo
+  }
+
+  def cloudFront(implicit config: Config, cloudFrontSettings: CloudFrontSetting) = cloudFrontSettings.cfClient(config)
+
+  type InvalidationResult = Future[Either[FailedInvalidation, SuccessfulInvalidation]]
+
+  type CloudFrontClientProvider = (Config) => AmazonCloudFront
+
+  case class SuccessfulInvalidation(invalidatedItemsCount: Int)(implicit pushOptions: PushOptions) extends SuccessReport {
+    def reportMessage = s"${Invalidated.renderVerb} ${invalidatedItemsCount ofType "item"} on CloudFront"
+  }
+
+  case class FailedInvalidation(error: Throwable)(implicit logger: Logger) extends ErrorReport {
+    def reportMessage = errorMessage(s"Failed to invalidate the CloudFront distribution", error)
+  }
+
+  def awsCloudFrontClient(config: Config) = new AmazonCloudFrontClient(awsCredentials(config))
+
+  def toInvalidationBatches(pushSuccessReports: Seq[PushSuccessReport])(implicit config: Config): Seq[InvalidationBatch] = {
+    def callerReference = s"s3_website gem ${System.currentTimeMillis()}"
+    if (config.cloudfront_wildcard_invalidation.contains(true) && pushSuccessReports.exists(needsInvalidation)) {
+      return Seq(new InvalidationBatch withPaths(new Paths withItems "/*" withQuantity 1) withCallerReference callerReference)
+    }
+    def defaultPath(paths: Seq[String]): Option[String] = {
+      // This is how we support the Default Root Object @ CloudFront (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DefaultRootObject.html)
+      // We could do this more accurately by fetching the distribution config (http://docs.aws.amazon.com/AmazonCloudFront/latest/APIReference/GetConfig.html)
+      // and reading the Default Root Object from there.
+      val containsPotentialDefaultRootObject = paths
+        .exists(
+          _
+            .replaceFirst("^/", "") // S3 keys do not begin with a slash
+            .contains("/") == false // See if the S3 key is a top-level key (i.e., it is not within a directory)
+        )
+      if (containsPotentialDefaultRootObject) Some("/") else None
+    }
+    val indexPath = config.cloudfront_invalidate_root collect {
+      case true if pushSuccessReports.nonEmpty => config.s3_key_prefix.map(prefix => s"/$prefix").getOrElse("") + "/index.html"
+    }
+
+    val invalidationPaths: Seq[String] = {
+      val paths = pushSuccessReports
+        .filter(needsInvalidation)
+        .map(toInvalidationPath)
+        .map(encodeUnsafeChars)
+        .map(applyInvalidateRootSetting)
+
+      val extraPathItems = defaultPath(paths) :: indexPath :: Nil collect {
+        case Some(path) => path
+      }
+
+      paths ++ extraPathItems
+    }
+
+    invalidationPaths
+      .grouped(1000) // CloudFront supports max 1000 invalidations in one request (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits)
+      .map { batchKeys =>
+        new InvalidationBatch() withPaths
+          (new Paths() withItems batchKeys withQuantity batchKeys.size) withCallerReference callerReference
+      }
+      .toSeq
+  }
+
+  def applyInvalidateRootSetting(path: String)(implicit config: Config) =
+    if (config.cloudfront_invalidate_root.contains(true))
+      path.replaceFirst("/index.html$", "/")
+    else
+      path
+
+  def toInvalidationPath(report: PushSuccessReport) = "/" + report.s3Key
+
+  def encodeUnsafeChars(invalidationPath: String) =
+    new URI("http", "cloudfront", invalidationPath, "")
+      .toURL
+      .getPath
+      .replaceAll("'", URLEncoder.encode("'", "UTF-8")) // CloudFront does not accept ' in invalidation path
+      .flatMap(chr => {
+        if (("[^\\x00-\\x7F]".r findFirstIn chr.toString).isDefined)
+          URLEncoder.encode(chr.toString, "UTF-8")
+        else
+          chr.toString
+      })
+
+  def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
+    case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) == FileUpdate
+    case SuccessfulDelete(_) => true
+    case _ => false
+  }
+
+  case class CloudFrontSetting(
+    cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
+    retryTimeUnit: TimeUnit = MINUTES
+  ) extends RetrySetting
+}

data/src/main/scala/s3/website/Logger.scala

@@ -0,0 +1,67 @@
+package s3.website
+
+class Logger(val verboseOutput: Boolean, onLog: Option[(String) => _] = None) {
+  def debug(msg: String) = if (verboseOutput) log(Debug, msg)
+  def info(msg: String) = log(Info, msg)
+  def fail(msg: String) = log(Failure, msg)
+  def warn(msg: String) = log(Warn, msg)
+
+  def info(report: SuccessReport) = log(Success, report.reportMessage)
+  def info(report: ErrorReport) = fail(report.reportMessage)
+
+  def pending(msg: String) = log(Wait, msg)
+
+  private def log(logType: LogType, msgRaw: String) = {
+    val msg = msgRaw.replaceAll("\\n", "\n       ") // Indent new lines, so that they arrange nicely with other log lines
+    val decoratedLogMessage = s"[$logType] $msg"
+    onLog.foreach(_(decoratedLogMessage))
+    println(decoratedLogMessage)
+  }
+
+  sealed trait LogType {
+    val prefix: String
+    override def toString = prefix
+  }
+  case object Debug extends LogType {
+    val prefix = "debg".cyan
+  }
+  case object Info extends LogType {
+    val prefix = "info".blue
+  }
+  case object Success extends LogType {
+    val prefix = "succ".green
+  }
+  case object Failure extends LogType {
+    val prefix = "fail".red
+  }
+  case object Wait extends LogType {
+    val prefix = "wait".yellow
+  }
+  case object Warn extends LogType {
+    val prefix = "warn".yellow
+  }
+
+  /**
+   * Idea copied from https://github.com/ktoso/scala-rainbow.
+   */
+  implicit class RainbowString(val s: String) {
+    import Console._
+
+    /** Colorize the given string foreground to ANSI black */
+    def black = BLACK + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def red = RED + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def green = GREEN + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def yellow = YELLOW + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def blue = BLUE + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def magenta = MAGENTA + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def cyan = CYAN + s + RESET
+    /** Colorize the given string foreground to ANSI red */
+    def white = WHITE + s + RESET
+  }
+}

data/src/main/scala/s3/website/Push.scala

@@ -0,0 +1,246 @@
+package s3.website
+
+import s3.website.model.Config.S3_website_yml
+import s3.website.model.Redirect.{Redirects, resolveRedirects}
+import s3.website.model.Site._
+import scala.concurrent.{ExecutionContextExecutor, Future, Await}
+import scala.concurrent.duration._
+import scala.language.postfixOps
+import s3.website.UploadHelper.{FutureUploads, resolveDeletes, resolveUploads}
+import s3.website.S3._
+import scala.concurrent.ExecutionContext.fromExecutor
+import java.util.concurrent.Executors.newFixedThreadPool
+import java.util.concurrent.ExecutorService
+import s3.website.model._
+import s3.website.model.FileUpdate
+import s3.website.model.NewFile
+import s3.website.S3.PushSuccessReport
+import scala.collection.mutable.ArrayBuffer
+import s3.website.CloudFront._
+import s3.website.S3.SuccessfulDelete
+import s3.website.CloudFront.SuccessfulInvalidation
+import s3.website.S3.S3Setting
+import s3.website.CloudFront.CloudFrontSetting
+import s3.website.S3.SuccessfulUpload
+import s3.website.CloudFront.FailedInvalidation
+import java.io.File
+import com.lexicalscope.jewel.cli.CliFactory.parseArguments
+import s3.website.ByteHelper.humanReadableByteCount
+import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
+
+object Push {
+
+  def main(args: Array[String]) {
+    implicit val cliArgs = parseArguments(classOf[CliArgs], args:_*)
+    implicit val logger: Logger = new Logger(cliArgs.verbose)
+    implicit val s3Settings = S3Setting()
+    implicit val cloudFrontSettings = CloudFrontSetting()
+    implicit val workingDirectory = new File(System.getProperty("user.dir")).getAbsoluteFile
+
+    System exit push
+  }
+
+  trait CliArgs {
+    import com.lexicalscope.jewel.cli.Option
+
+    @Option(defaultToNull = true) def site: String
+    @Option(longName = Array("config-dir"), defaultToNull = true) def configDir: String
+    @Option def verbose: Boolean
+    @Option(longName = Array("dry-run")) def dryRun: Boolean
+    @Option(longName = Array("force")) def force: Boolean
+  }
+
+  def push(implicit cliArgs: CliArgs, s3Settings: S3Setting, cloudFrontSettings: CloudFrontSetting, workingDirectory: File, logger: Logger): ExitCode = {
+    implicit val pushOptions = new PushOptions {
+      def dryRun = cliArgs.dryRun
+      def force  = cliArgs.force
+    }
+
+    implicit val yamlConfig = S3_website_yml(new File(Option(cliArgs.configDir).getOrElse(workingDirectory.getPath) + "/s3_website.yml"))
+
+    val errorOrPushStatus = for (
+      loadedSite <- loadSite.right
+    ) yield {
+      implicit val site = loadedSite
+      val threadPool = newFixedThreadPool(site.config.concurrency_level)
+      implicit val executor = fromExecutor(threadPool)
+      val pushStatus = pushSite
+      threadPool.shutdownNow()
+      pushStatus
+    }
+
+    errorOrPushStatus.left foreach (err => logger.fail(s"Could not load the site: ${err.reportMessage}"))
+    errorOrPushStatus fold((err: ErrorReport) => 1, pushStatus => pushStatus)
+  }
+
+  def pushSite(
+                implicit site: Site,
+                executor: ExecutionContextExecutor,
+                s3Settings: S3Setting,
+                cloudFrontSettings: CloudFrontSetting,
+                logger: Logger,
+                pushOptions: PushOptions
+                ): ExitCode = {
+    logger.info(s"${Deploy.renderVerb} ${site.rootDirectory}/* to ${site.config.s3_bucket}")
+    val s3FilesFuture = resolveS3Files()
+    val redirectsFuture = resolveRedirects(s3FilesFuture)
+    val redirectReports = redirectsFuture.map { errOrRedirects =>
+      errOrRedirects.right.map(_.filter(_.needsUpload).map(S3 uploadRedirect _))
+    }
+
+    val uploadReports = for {
+      errorOrUploads <- resolveUploads(s3FilesFuture)
+    } yield errorOrUploads.right.map(_.map(S3 uploadFile _))
+
+    val deleteReports = redirectsFuture flatMap { errOrRedirects =>
+      errOrRedirects.fold(
+        err => Future(Left(err)),
+        redirects => resolveDeletes(s3FilesFuture, redirects)
+      )
+    } map { deletes =>
+      deletes.right.map(keysToDelete => keysToDelete.map(S3 delete _))
+    }
+
+    val allReports = Future.sequence(redirectReports :: uploadReports :: deleteReports :: Nil) map { reports =>
+      reports.foldLeft(Nil: PushReports) { (memo, report: Either[ErrorReport, Seq[Future[PushErrorOrSuccess]]]) =>
+        report match {
+          case Left(err) =>
+            memo :+ Left(err)
+          case Right(pushResults: Seq[Future[PushErrorOrSuccess]]) =>
+            memo ++ pushResults.map(Right(_))
+        }
+      }
+    }
+    val finishedPushOps = awaitForResults(Await.result(allReports, 1 day))
+    val invalidationSucceeded = invalidateCloudFrontItems(finishedPushOps)
+    
+    report(finishedPushOps, invalidationSucceeded)
+  }
+  
+  def invalidateCloudFrontItems
+    (finishedPushOperations: FinishedPushOperations)
+    (implicit config: Config, cloudFrontSettings: CloudFrontSetting, ec: ExecutionContextExecutor, logger: Logger, pushOptions: PushOptions):
+  Option[InvalidationSucceeded] =
+    config.cloudfront_distribution_id.map { distributionId =>
+      val pushSuccessReports = 
+        finishedPushOperations.map {
+          ops =>
+            for {
+              failedOrSucceededPushes <- ops.right
+              successfulPush <- failedOrSucceededPushes.right
+            } yield successfulPush
+        }.foldLeft(Seq(): Seq[PushSuccessReport]) {
+          (reports, failOrSucc) =>
+            failOrSucc.fold(
+              _ => reports,
+              (pushSuccessReport: PushSuccessReport) => reports :+ pushSuccessReport
+            )
+        }
+      val invalidationResults: Seq[Either[FailedInvalidation, SuccessfulInvalidation]] =
+        toInvalidationBatches(pushSuccessReports) map { invalidationBatch =>
+          Await.result(
+            CloudFront.invalidate(invalidationBatch, distributionId),
+            atMost = 1 day
+          )
+        }
+      if (invalidationResults.exists(_.isLeft))
+        false // If one of the invalidations failed, mark the whole process as failed
+      else
+        true
+    }
+
+  type InvalidationSucceeded = Boolean
+
+  def report(finishedPushOps: FinishedPushOperations, invalidationSucceeded: Option[Boolean])
+                       (implicit config: Config, logger: Logger, pushOptions: PushOptions): ExitCode = {
+    val pushCounts = resolvePushCounts(finishedPushOps)
+    logger.info(s"Summary: ${pushCountsToString(pushCounts)}")
+    val pushOpExitCode = finishedPushOps.foldLeft(0) { (memo, finishedUpload) =>
+      memo + finishedUpload.fold(
+        (error: ErrorReport) => 1,
+        (failedOrSucceededUpload: Either[PushFailureReport, PushSuccessReport]) =>
+          if (failedOrSucceededUpload.isLeft) 1 else 0
+      )
+    } min 1
+    val cloudFrontInvalidationExitCode = invalidationSucceeded.fold(0)(allInvalidationsSucceeded =>
+      if (allInvalidationsSucceeded) 0 else 1
+    )
+
+    val exitCode = (pushOpExitCode + cloudFrontInvalidationExitCode) min 1
+
+    exitCode match {
+      case 0 if !pushOptions.dryRun && pushCounts.thereWasSomethingToPush =>
+        logger.info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
+      case 1 =>
+        logger.fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
+      case _ =>
+    }
+    exitCode
+  }
+
+  def awaitForResults(uploadReports: PushReports)(implicit executor: ExecutionContextExecutor): FinishedPushOperations =
+    uploadReports map (_.right.map {
+      rep => Await.result(rep, 1 day)
+    })
+
+  def resolvePushCounts(implicit finishedOperations: FinishedPushOperations) = finishedOperations.foldLeft(PushCounts()) {
+    (counts: PushCounts, uploadReport) =>
+      uploadReport.fold(
+        (error: ErrorReport) => counts.copy(failures = counts.failures + 1),
+        failureOrSuccess => failureOrSuccess.fold(
+          (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
+          (successReport: PushSuccessReport) =>
+            successReport match {
+              case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) match {
+                case NewFile      => counts.copy(newFiles = counts.newFiles + 1).addTransferStats(succ) // TODO nasty repetition here
+                case FileUpdate   => counts.copy(updates = counts.updates + 1).addTransferStats(succ)
+                case RedirectFile => counts.copy(redirects = counts.redirects + 1).addTransferStats(succ)
+              }
+            case SuccessfulDelete(_) => counts.copy(deletes = counts.deletes + 1)
+          }
+        )
+      )
+  }
+
+  def pushCountsToString(pushCounts: PushCounts)(implicit pushOptions: PushOptions): String =
+    pushCounts match {
+      case PushCounts(updates, newFiles, failures, redirects, deletes, _, _)
+        if updates == 0 && newFiles == 0 && failures == 0 && redirects == 0 && deletes == 0 =>
+          PushNothing.renderVerb
+      case PushCounts(updates, newFiles, failures, redirects, deletes, uploadedBytes, uploadDurations) =>
+        val reportClauses: scala.collection.mutable.ArrayBuffer[String] = ArrayBuffer()
+        if (updates > 0)       reportClauses += s"${Updated.renderVerb} ${updates ofType "file"}."
+        if (newFiles > 0)      reportClauses += s"${Created.renderVerb} ${newFiles ofType "file"}."
+        if (failures > 0)      reportClauses += s"${failures ofType "operation"} failed." // This includes both failed uploads and deletes.
+        if (redirects > 0)     reportClauses += s"${Applied.renderVerb} ${redirects ofType "redirect"}."
+        if (deletes > 0)       reportClauses += s"${Deleted.renderVerb} ${deletes ofType "file"}."
+        if (uploadedBytes > 0) {
+          val transferSuffix = humanizeUploadSpeed(uploadedBytes, uploadDurations: _*).fold(".")(speed => s", $speed.")
+          reportClauses += s"${Transferred.renderVerb} ${humanReadableByteCount(uploadedBytes)}$transferSuffix"
+        }
+        reportClauses.mkString(" ")
+    }
+
+  case class PushCounts(
+                         updates: Int = 0, 
+                         newFiles: Int = 0, 
+                         failures: Int = 0, 
+                         redirects: Int = 0, 
+                         deletes: Int = 0,
+                         uploadedBytes: Long = 0,
+                         uploadDurations: Seq[UploadDuration] = Nil
+                         ) {
+    val thereWasSomethingToPush = updates + newFiles + redirects + deletes > 0
+
+    def addTransferStats(successfulUpload: SuccessfulUpload): PushCounts =
+      copy(
+        uploadedBytes = uploadedBytes + (successfulUpload.uploadSize getOrElse 0L),
+        uploadDurations = uploadDurations ++ successfulUpload.details.fold(_.uploadDuration, _ => None)
+      )
+  }
+
+  type FinishedPushOperations = Seq[Either[ErrorReport, PushErrorOrSuccess]]
+  type PushReports = Seq[Either[ErrorReport, Future[PushErrorOrSuccess]]]
+  case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
+  type ExitCode = Int
+}

data/src/main/scala/s3/website/Ruby.scala

@@ -0,0 +1,14 @@
+package s3.website
+
+object Ruby {
+  lazy val rubyRuntime = org.jruby.Ruby.newInstance() // Instantiate heavy object
+
+  def rubyRegexMatches(text: String, regex: String) = {
+    val z  = rubyRuntime.evalScriptlet(
+      s"""# encoding: utf-8
+          !!Regexp.new("$regex").match("$text")"""
+    )
+    z.toJava(classOf[Boolean]).asInstanceOf[Boolean]
+  }
+
+}