s3_website_monadic 0.0.1

data/spec/lib/upload_spec.rb

@@ -0,0 +1,303 @@
+require 'spec_helper'
+
+describe S3Website::Upload do
+  describe 'uploading blacklisted files' do
+    let(:blacklisted_files) {
+      [ 's3_website.yml' ]
+    }
+    it 'should fail if the upload file is s3_website.yml' do
+      blacklisted_files.each do |blacklisted_file|
+        expect {
+          S3Website::Upload.new blacklisted_file, mock(), {}, mock()
+        }.to raise_error "May not upload #{blacklisted_file}, because it's blacklisted"
+      end
+    end
+
+    it 'should fail to upload configured blacklisted files' do
+      config = { 'exclude_from_upload' => 'vendor' }
+
+      expect {
+        S3Website::Upload.new "vendor/jquery/development.js", mock(), config, mock()
+      }.to raise_error "May not upload vendor/jquery/development.js, because it's blacklisted"
+    end
+
+    context 'the uploaded file matches a value in the exclude_from_upload setting' do
+      it 'should fail to upload any configured blacklisted files' do
+        config = { 'exclude_from_upload' => ['vendor', 'tests'] }
+
+        expect {
+          S3Website::Upload.new "vendor/jquery/development.js", mock(), config, mock()
+        }.to raise_error "May not upload vendor/jquery/development.js, because it's blacklisted"
+
+        expect {
+          S3Website::Upload.new "tests/spec_helper.js", mock(), config, mock()
+        }.to raise_error "May not upload tests/spec_helper.js, because it's blacklisted"
+      end
+
+      it 'supports regexes in the exclude_from_upload setting' do
+        config = { 'exclude_from_upload' => 'test.*' }
+
+        expect {
+          S3Website::Upload.new "tests/spec_helper.js", mock(), config, mock()
+        }.to raise_error "May not upload tests/spec_helper.js, because it's blacklisted"
+      end
+    end
+  end
+
+  describe 'reduced redundancy setting' do
+    let(:config) {
+      { 's3_reduced_redundancy' => true }
+    }
+
+    it 'allows storing a file under the Reduced Redundancy Storage' do
+      should_upload(
+        file = 'index.html',
+        site = 'features/support/test_site_dirs/my.blog.com/_site', config) { |s3_object|
+        s3_object.should_receive(:write).with(
+          anything(),
+          include(:reduced_redundancy => true)
+        )
+      }
+    end
+  end
+
+  describe 'content type resolving' do
+    it 'adds the content type of the uploaded CSS file into the S3 object' do
+      should_upload(
+        file = 'css/styles.css',
+        site = 'features/support/test_site_dirs/my.blog.com/_site') { |s3_object|
+        s3_object.should_receive(:write).with(
+          anything(),
+          include(:content_type => 'text/css; charset=utf-8')
+        )
+      }
+    end
+
+    it 'adds the content type of the uploaded HTML file into the S3 object' do
+      should_upload(
+        file = 'index.html',
+        site = 'features/support/test_site_dirs/my.blog.com/_site') { |s3_object|
+        s3_object.should_receive(:write).with(
+          anything(),
+          include(:content_type => 'text/html; charset=utf-8')
+        )
+      }
+    end
+
+    describe 'encoding of text documents' do
+      it 'should mark all text documents as utf-8' do
+        should_upload(
+          file = 'file.txt',
+          site = 'features/support/test_site_dirs/site-with-text-doc.com/_site') { |s3_object|
+          s3_object.should_receive(:write).with(
+            anything(),
+            include(:content_type => 'text/plain; charset=utf-8')
+          )
+        }
+      end
+    end
+
+    context 'the user specifies a mime-type for extensionless files' do
+      let(:config) {{
+        'extensionless_mime_type' => "text/html"
+      }}
+
+      it 'adds the content type of the uploaded extensionless file into the S3 object' do
+        should_upload(
+          file = 'index',
+          site = 'features/support/test_site_dirs/my.blog-with-clean-urls.com/_site',
+          config) { |s3_object|
+          s3_object.should_receive(:write).with(
+            anything(),
+            include(:content_type => 'text/html; charset=utf-8')
+          )
+        }
+      end
+    end
+  end  
+
+  describe 'gzip compression' do
+    let(:config){
+      {
+        's3_reduced_redundancy' => false,
+        'gzip' => true
+      }
+    }
+
+    subject{ S3Website::Upload.new("index.html", mock(), config, 'features/support/test_site_dirs/my.blog.com/_site') }
+
+    describe '#gzip?' do
+      it 'should be false if the config does not specify gzip' do
+        config.delete 'gzip'
+        subject.should_not be_gzip
+      end
+
+      it 'should be false if gzip is true but does not match a default extension' do
+        subject.stub(:path).and_return("index.bork")
+        subject.should_not be_gzip
+      end
+
+      it 'should be true if gzip is true and file extension matches' do
+        subject.should be_gzip
+      end
+
+      it 'should be true if gzip is true and file extension matches custom supplied' do
+        config['gzip'] = %w(.bork)
+        subject.stub(:path).and_return('index.bork')
+        subject.should be_gzip
+      end
+    end       
+
+    describe '#gzipped_file' do
+      it 'should return a gzipped version of the file' do
+        gz = Zlib::GzipReader.new(subject.send(:gzipped_file))
+        gz.read.should == File.read('features/support/test_site_dirs/my.blog.com/_site/index.html')
+      end
+    end
+  end
+  
+  describe 'gzip zopfli' do
+    let(:config){
+      {
+        's3_reduced_redundancy' => false,
+        'gzip' => true,
+        'gzip_zopfli' => true
+      }
+    }
+
+    subject{ S3Website::Upload.new("index.html", mock(), config, 'features/support/test_site_dirs/my.blog.com/_site') }
+
+    # Zopfli should be compatible with the gzip format
+    describe '#gzipped_file' do
+      it 'should return a gzipped version of the file' do
+        gz = Zlib::GzipReader.new(subject.send(:gzipped_file))
+        gz.read.should == File.read('features/support/test_site_dirs/my.blog.com/_site/index.html')
+      end
+    end
+  end
+
+  describe 'cache control' do
+    let(:config){
+      {
+        's3_reduced_redundancy' => false,
+        'max_age' => 300
+      }
+    }
+
+    let(:subject) {
+      S3Website::Upload.new(
+        "index.html",
+        mock(),
+        config,
+        'features/support/test_site_dirs/my.blog.com/_site'
+      )
+    }
+
+    describe '#cache_control?' do
+      it 'should be false if max_age is missing' do
+        config.delete 'max_age'
+        subject.should_not be_cache_control
+      end
+
+      it 'should be true if max_age is present' do
+        subject.should be_cache_control
+      end
+
+      it 'should be true if max_age is a hash' do
+        config['max_age'] = {'*' => 300}
+        subject.should be_cache_control
+      end
+    end
+
+    context 'the user specifies max-age as zero' do
+      let(:config) {{
+        'max_age' => 0
+      }}
+
+      it 'includes the no-cache declaration in the cache-control metadata' do
+        subject.send(:upload_options)[:cache_control].should == 'no-cache, max-age=0'
+      end
+    end
+
+    describe '#max_age' do
+      it 'should be the universal value if one is set' do
+        subject.send(:max_age).should == 300
+      end
+
+      it 'should be the file-specific value if one is set' do
+        config['max_age'] = {'*index.html' => 500}
+        subject.send(:max_age).should == 500
+      end
+
+      it 'should be zero if no file-specific value hit' do
+        config['max_age'] = {'*.js' => 500}
+        subject.send(:max_age).should == 0
+      end
+
+      context 'overriding the more general setting with the more specific' do
+        let(:config){
+          {
+            's3_reduced_redundancy' => false,
+            'max_age' => {
+              '**'        => 150,
+              'assets/**' => 86400
+            }
+          }
+        }
+
+        it 'respects the most specific max-age selector' do
+          subject = S3Website::Upload.new(
+            'assets/picture.gif',
+            mock(),
+            config,
+            'features/support/test_site_dirs/index-and-assets.blog.fi/_site'
+          )
+          subject.send(:max_age).should == 86400
+        end
+
+        it 'respects the most specific max-age selector' do
+          subject = S3Website::Upload.new(
+            'index.html',
+            mock(),
+            config,
+            'features/support/test_site_dirs/index-and-assets.blog.fi/_site'
+          )
+          subject.send(:max_age).should == 150
+        end
+      end
+    end
+  end
+
+  def should_upload(file_to_upload, site_dir, config = {})
+    def create_verifying_s3_client(file_to_upload, &block)
+      def create_objects(file_to_upload, &block)
+        def create_html_s3_object(file_to_upload, &block)
+          s3_object = stub('s3_object')
+          yield s3_object
+          s3_object
+        end
+        objects = {}
+        objects[file_to_upload] = create_html_s3_object(file_to_upload, &block)
+        objects
+      end
+      def create_bucket(file_to_upload, &block)
+        bucket = stub('bucket')
+        bucket.stub(:objects => create_objects(file_to_upload, &block))
+        bucket
+      end
+      buckets = stub('buckets')
+      buckets.stub(:[] => create_bucket(file_to_upload, &block))
+      s3 = stub('s3')
+      s3.stub(:buckets => buckets)
+      s3
+    end
+
+    s3_client = create_verifying_s3_client(file_to_upload) do |s3_object|
+      yield s3_object
+    end
+    S3Website::Upload.new(file_to_upload,
+                          s3_client,
+                          config,
+                          site_dir).perform!
+  end
+end

data/spec/lib/uploader_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe S3Website::Uploader do
+  context '#load_all_local_files' do
+    let(:files) {
+      S3Website::Uploader.send(:load_all_local_files,
+                                'spec/sample_files/hyde_site/_site')
+    }
+
+    it 'loads regular files' do
+      files.should include('css/styles.css')
+      files.should include('index.html')
+    end
+
+    it 'loads also dotfiles' do
+      files.should include('.vimrc')
+    end
+  end
+
+  context "honoring the ignore_on_server setting" do
+    it "ignores files which match a regular expression" do
+      files_to_delete = S3Website::Uploader.build_list_of_files_to_delete(["a", "b", "ignored"], ["a"], "ignored")
+      files_to_delete.should eq ["b"]
+    end
+
+    it "let's the user specify the regexes in a list" do
+      files_to_delete = S3Website::Uploader.build_list_of_files_to_delete(["a", "b", "ignored"], ["a"], ["ignored"])
+      files_to_delete.should eq ["b"]
+    end
+
+
+    it "does not ignore when you don't provide an ignored regex" do
+      files_to_delete = S3Website::Uploader.build_list_of_files_to_delete(["a", "b", "ignored"], ["a"])
+      files_to_delete.should eq ["b", "ignored"]
+    end
+  end
+end

data/spec/sample_files/hyde_site/_site/.vimrc

@@ -0,0 +1,5 @@
+" Disable the arrow keys in the normal mode – using them there only slows you down. Use hkjl instead.
+map <Left> :echo "no!"<cr>
+map <Right> :echo "no!"<cr>
+map <Up> :echo "no!"<cr>
+map <Down> :echo "no!"<cr>

data/spec/sample_files/hyde_site/_site/css/styles.css

@@ -0,0 +1,3 @@
+body {
+
+}

data/spec/sample_files/hyde_site/_site/index.html

@@ -0,0 +1 @@
+<div>Hello world</div>

data/spec/sample_files/hyde_site/s3_website.yml

@@ -0,0 +1,3 @@
+s3_id: <%= 'hello' %>
+s3_secret: <%= 'world' %>
+s3_bucket: galaxy

data/spec/sample_files/tokyo_site/_site/.vimrc

@@ -0,0 +1,5 @@
+" Disable the arrow keys in the normal mode – using them there only slows you down. Use hkjl instead.
+map <Left> :echo "no!"<cr>
+map <Right> :echo "no!"<cr>
+map <Up> :echo "no!"<cr>
+map <Down> :echo "no!"<cr>

data/spec/sample_files/tokyo_site/_site/css/styles.css

@@ -0,0 +1,3 @@
+body {
+
+}

data/spec/sample_files/tokyo_site/_site/index.html

@@ -0,0 +1 @@
+<div>Hello world</div>

data/spec/sample_files/tokyo_site/s3_website.yml

@@ -0,0 +1,4 @@
+s3_id: <%= 'hello' %>
+s3_secret: <%= 'world' %>
+s3_bucket: galaxy
+s3_endpoint: ap-northeast-1

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require File.dirname(__FILE__) + "/../lib/s3_website.rb"

data/src/main/scala/s3/website/CloudFront.scala

@@ -0,0 +1,96 @@
+package s3.website
+
+import s3.website.model.{Redirect, Config}
+import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
+import s3.website.CloudFront.{FailedInvalidation, SuccessfulInvalidation, CloudFrontClientProvider}
+import scala.util.{Failure, Success, Try}
+import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
+import scala.collection.JavaConversions._
+import scala.concurrent.duration._
+import s3.website.S3.{SuccessfulUpload, PushSuccessReport}
+import com.amazonaws.auth.BasicAWSCredentials
+
+class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUnit) {
+
+  def invalidate(invalidationBatch: InvalidationBatch, distributionId: String)(implicit config: Config): InvalidationResult = {
+    def tryInvalidate(implicit attempt: Int = 1): Try[SuccessfulInvalidation] =
+      Try {
+        val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
+        cfClient(config).createInvalidation(invalidationReq)
+        val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
+        println(s"Invalidated ${result.invalidatedItemsCount} item(s) on the CloudFront distribution $distributionId.")
+        result
+      } recoverWith {
+        case e: TooManyInvalidationsInProgressException =>
+          implicit val duration: Duration = Duration(
+            (fibs drop attempt).head min 15, /* AWS docs way that invalidations complete in 15 minutes */
+            sleepUnit
+          )
+          println(maxInvalidationsExceededInfo)
+          Thread.sleep(duration.toMillis)
+          tryInvalidate(attempt + 1)
+      }
+
+    tryInvalidate() match {
+      case Success(res) =>
+        Right(res)
+      case Failure(err) =>
+        println(s"Failed to invalidate the CloudFront distribution $distributionId (${err.getMessage})")
+        Left(FailedInvalidation())
+    }
+  }
+
+  def maxInvalidationsExceededInfo(implicit sleepDuration: Duration, attempt: Int) = {
+    val basicInfo = s"The maximum amount of CloudFront invalidations has exceeded. Trying again in $sleepDuration, please wait."
+    val extendedInfo =
+      s"""|$basicInfo
+          |  For more information, see http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits"""
+        .stripMargin
+    if (attempt == 1)
+      extendedInfo
+    else
+      basicInfo
+  }
+
+  type InvalidationResult = Either[FailedInvalidation, SuccessfulInvalidation]
+
+  lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
+}
+
+object CloudFront {
+
+  type CloudFrontClientProvider = (Config) => AmazonCloudFront
+
+  case class SuccessfulInvalidation(invalidatedItemsCount: Int)
+
+  case class FailedInvalidation()
+
+  def awsCloudFrontClient(config: Config) =
+    new AmazonCloudFrontClient(new BasicAWSCredentials(config.s3_id, config.s3_secret))
+  
+  def toInvalidationBatches(pushSuccessReports: Seq[PushSuccessReport])(implicit config: Config): Seq[InvalidationBatch] =
+    pushSuccessReports
+      .filterNot(isRedirect) // Assume that redirect objects are never cached.
+      .map("/" + _.s3Key) // CloudFront keys always have the slash in front
+      .map { path =>
+        if (config.cloudfront_invalidate_root.exists(_ == true))
+          path.replaceFirst("/index.html$", "/")
+        else
+          path
+      }
+      .grouped(1000) // CloudFront supports max 1000 invalidations in one request (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits)
+      .map { batchKeys =>
+        new InvalidationBatch() withPaths
+          (new Paths() withItems batchKeys withQuantity batchKeys.size) withCallerReference
+            s"s3_website gem ${System.currentTimeMillis()}"
+      }
+      .toSeq
+
+  def isRedirect: PartialFunction[PushSuccessReport, Boolean] = {
+    case SuccessfulUpload(upload) => upload.uploadType match {
+      case Redirect => true
+      case _ => false
+    }
+    case _ => false
+  }
+}

data/src/main/scala/s3/website/Diff.scala

@@ -0,0 +1,42 @@
+package s3.website
+
+import s3.website.model._
+import s3.website.Ruby.rubyRegexMatches
+
+object Diff {
+
+  def resolveDeletes(localFiles: Seq[LocalFile], s3Files: Seq[S3File], redirects: Seq[Upload with UploadTypeResolved])(implicit config: Config): Seq[S3File] = {
+    val keysNotToBeDeleted: Set[String] = (localFiles ++ redirects).map(_.s3Key).toSet
+    s3Files.filterNot { s3File =>
+      val ignoreOnServer = config.ignore_on_server.exists(_.fold(
+        (ignoreRegex: String)        => rubyRegexMatches(s3File.s3Key, ignoreRegex),
+        (ignoreRegexes: Seq[String]) => ignoreRegexes.exists(rubyRegexMatches(s3File.s3Key, _))
+      ))
+      keysNotToBeDeleted.exists(_ == s3File.s3Key) || ignoreOnServer
+    }
+  }
+
+  def resolveUploads(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
+  Stream[Either[Error, Upload with UploadTypeResolved]] = {
+    val remoteS3KeysIndex = s3Files.map(_.s3Key).toSet
+    val remoteMd5Index = s3Files.map(_.md5).toSet
+    localFiles
+      .toStream // Load lazily, because the MD5 computation for the local file requires us to read the whole file
+      .map(resolveUploadSource)
+      .collect {
+      case errorOrUpload if errorOrUpload.right.exists(isNewUpload(remoteS3KeysIndex)) =>
+        for (upload <- errorOrUpload.right) yield upload withUploadType NewFile
+      case errorOrUpload if errorOrUpload.right.exists(isUpdate(remoteS3KeysIndex, remoteMd5Index)) =>
+        for (upload <- errorOrUpload.right) yield upload withUploadType Update
+    }
+  }
+
+  def isNewUpload(remoteS3KeysIndex: Set[String])(u: Upload) = !remoteS3KeysIndex.exists(_ == u.s3Key)
+
+  def isUpdate(remoteS3KeysIndex: Set[String], remoteMd5Index: Set[String])(u: Upload) =
+    remoteS3KeysIndex.exists(_ == u.s3Key) && !remoteMd5Index.exists(remoteMd5 => u.essence.right.exists(_.md5 == remoteMd5))
+
+  def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[Error, Upload] =
+    for (upload <- LocalFile.toUpload(localFile).right)
+    yield upload
+}

data/src/main/scala/s3/website/Implicits.scala

@@ -0,0 +1,7 @@
+package s3.website
+
+import s3.website.model.{Site, Config}
+
+object Implicits {
+  implicit def site2Config(implicit site: Site): Config = site.config
+}