s3_website_monadic 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +10 -0
- data/.travis.yml +3 -0
- data/Gemfile +3 -0
- data/LICENSE +42 -0
- data/README.md +451 -0
- data/Rakefile +24 -0
- data/additional-docs/example-configurations.md +62 -0
- data/additional-docs/setting-up-aws-credentials.md +51 -0
- data/assembly.sbt +3 -0
- data/bin/s3_website +80 -0
- data/build.sbt +33 -0
- data/changelog.md +215 -0
- data/features/as-library.feature +29 -0
- data/features/cassettes/cucumber_tags/create-redirect.yml +384 -0
- data/features/cassettes/cucumber_tags/empty-bucket.yml +89 -0
- data/features/cassettes/cucumber_tags/new-and-changed-files.yml +303 -0
- data/features/cassettes/cucumber_tags/new-files-for-sydney.yml +211 -0
- data/features/cassettes/cucumber_tags/new-files.yml +355 -0
- data/features/cassettes/cucumber_tags/no-new-or-changed-files.yml +359 -0
- data/features/cassettes/cucumber_tags/one-file-to-delete.yml +390 -0
- data/features/cassettes/cucumber_tags/only-changed-files.yml +411 -0
- data/features/cassettes/cucumber_tags/s3-and-cloudfront-after-deleting-a-file.yml +434 -0
- data/features/cassettes/cucumber_tags/s3-and-cloudfront-when-updating-a-file.yml +435 -0
- data/features/cassettes/cucumber_tags/s3-and-cloudfront.yml +290 -0
- data/features/cloudfront.feature +54 -0
- data/features/command-line-help.feature +54 -0
- data/features/delete.feature +19 -0
- data/features/error_reporting.feature +24 -0
- data/features/instructions-for-new-user.feature +154 -0
- data/features/jekyll-support.feature +20 -0
- data/features/nanoc-support.feature +20 -0
- data/features/push.feature +115 -0
- data/features/redirects.feature +14 -0
- data/features/security.feature +15 -0
- data/features/step_definitions/steps.rb +86 -0
- data/features/support/env.rb +26 -0
- data/features/support/test_site_dirs/cdn-powered.blog.fi/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/cdn-powered.blog.fi/_site/index.html +5 -0
- data/features/support/test_site_dirs/cdn-powered.blog.fi/s3_website.yml +4 -0
- data/features/support/test_site_dirs/cdn-powered.when-deleted-a-file.blog.fi/_site/index.html +10 -0
- data/features/support/test_site_dirs/cdn-powered.when-deleted-a-file.blog.fi/s3_website.yml +5 -0
- data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/_site/index.html +10 -0
- data/features/support/test_site_dirs/cdn-powered.with-one-change.blog.fi/s3_website.yml +4 -0
- data/features/support/test_site_dirs/create-redirects/_site/.gitkeep +0 -0
- data/features/support/test_site_dirs/create-redirects/s3_website.yml +6 -0
- data/features/support/test_site_dirs/ignored-files.com/_site/css/styles.css +4 -0
- data/features/support/test_site_dirs/ignored-files.com/_site/index.html +8 -0
- data/features/support/test_site_dirs/ignored-files.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/assets/picture.gif +0 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/_site/index.html +5 -0
- data/features/support/test_site_dirs/index-and-assets.blog.fi/s3_website.yml +3 -0
- data/features/support/test_site_dirs/jekyllrb.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/jekyllrb.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/jekyllrb.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/my.blog-with-clean-urls.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/my.blog-with-clean-urls.com/_site/index +5 -0
- data/features/support/test_site_dirs/my.blog-with-clean-urls.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/my.blog.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/my.blog.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/my.blog.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/my.sydney.blog.au/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/my.sydney.blog.au/_site/index.html +5 -0
- data/features/support/test_site_dirs/my.sydney.blog.au/s3_website.yml +4 -0
- data/features/support/test_site_dirs/nanoc.ws/public/output/css/styles.css +3 -0
- data/features/support/test_site_dirs/nanoc.ws/public/output/index.html +5 -0
- data/features/support/test_site_dirs/nanoc.ws/s3_website.yml +3 -0
- data/features/support/test_site_dirs/new-and-changed-files.com/_site/css/styles.css +4 -0
- data/features/support/test_site_dirs/new-and-changed-files.com/_site/index.html +8 -0
- data/features/support/test_site_dirs/new-and-changed-files.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/no-new-or-changed-files.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/no-new-or-changed-files.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/no-new-or-changed-files.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/only-changed-files.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/only-changed-files.com/_site/index.html +9 -0
- data/features/support/test_site_dirs/only-changed-files.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/site-that-contains-s3-website-file.com/_site/s3_website.yml +3 -0
- data/features/support/test_site_dirs/site-that-contains-s3-website-file.com/s3_website.yml +3 -0
- data/features/support/test_site_dirs/site-with-text-doc.com/_site/file.txt +1 -0
- data/features/support/test_site_dirs/site.with.css-maxage.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.css-maxage.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.css-maxage.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-and-max-aged-content.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.gzipped-html.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.gzipped-html.com/s3_website.yml +5 -0
- data/features/support/test_site_dirs/site.with.maxage.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/site.with.maxage.com/_site/index.html +5 -0
- data/features/support/test_site_dirs/site.with.maxage.com/s3_website.yml +4 -0
- data/features/support/test_site_dirs/unpublish-a-post.com/_site/css/styles.css +3 -0
- data/features/support/test_site_dirs/unpublish-a-post.com/s3_website.yml +3 -0
- data/features/support/vcr.rb +20 -0
- data/features/website-performance.feature +57 -0
- data/lib/cloudfront/invalidator.rb +37 -0
- data/lib/s3_website/config_loader.rb +55 -0
- data/lib/s3_website/diff_helper.rb +113 -0
- data/lib/s3_website/endpoint.rb +37 -0
- data/lib/s3_website/errors.rb +42 -0
- data/lib/s3_website/jekyll.rb +5 -0
- data/lib/s3_website/keyboard.rb +27 -0
- data/lib/s3_website/nanoc.rb +5 -0
- data/lib/s3_website/parallelism.rb +25 -0
- data/lib/s3_website/paths.rb +39 -0
- data/lib/s3_website/retry.rb +19 -0
- data/lib/s3_website/tasks.rb +36 -0
- data/lib/s3_website/upload.rb +137 -0
- data/lib/s3_website/uploader.rb +177 -0
- data/lib/s3_website.rb +34 -0
- data/project/assembly.sbt +1 -0
- data/project/build.properties +0 -0
- data/project/plugins.sbt +1 -0
- data/project/sbt-launch-0.13.2.jar +0 -0
- data/resources/configuration_file_template.yml +56 -0
- data/s3_website.gemspec +41 -0
- data/sbt +4 -0
- data/spec/lib/cloudfront/invalidator_spec.rb +60 -0
- data/spec/lib/config_loader_spec.rb +20 -0
- data/spec/lib/endpoint_spec.rb +31 -0
- data/spec/lib/error_spec.rb +21 -0
- data/spec/lib/keyboard_spec.rb +62 -0
- data/spec/lib/parallelism_spec.rb +81 -0
- data/spec/lib/paths_spec.rb +7 -0
- data/spec/lib/retry_spec.rb +34 -0
- data/spec/lib/upload_spec.rb +303 -0
- data/spec/lib/uploader_spec.rb +37 -0
- data/spec/sample_files/hyde_site/_site/.vimrc +5 -0
- data/spec/sample_files/hyde_site/_site/css/styles.css +3 -0
- data/spec/sample_files/hyde_site/_site/index.html +1 -0
- data/spec/sample_files/hyde_site/s3_website.yml +3 -0
- data/spec/sample_files/tokyo_site/_site/.vimrc +5 -0
- data/spec/sample_files/tokyo_site/_site/css/styles.css +3 -0
- data/spec/sample_files/tokyo_site/_site/index.html +1 -0
- data/spec/sample_files/tokyo_site/s3_website.yml +4 -0
- data/spec/spec_helper.rb +1 -0
- data/src/main/scala/s3/website/CloudFront.scala +96 -0
- data/src/main/scala/s3/website/Diff.scala +42 -0
- data/src/main/scala/s3/website/Implicits.scala +7 -0
- data/src/main/scala/s3/website/Push.scala +191 -0
- data/src/main/scala/s3/website/Ruby.scala +12 -0
- data/src/main/scala/s3/website/S3.scala +139 -0
- data/src/main/scala/s3/website/model/Config.scala +152 -0
- data/src/main/scala/s3/website/model/S3Endpoint.scala +22 -0
- data/src/main/scala/s3/website/model/Site.scala +68 -0
- data/src/main/scala/s3/website/model/errors.scala +11 -0
- data/src/main/scala/s3/website/model/push.scala +192 -0
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +445 -0
- metadata +508 -0
@@ -0,0 +1,192 @@
|
|
1
|
+
package s3.website.model
|
2
|
+
|
3
|
+
import com.amazonaws.services.s3.model.S3ObjectSummary
|
4
|
+
import java.io._
|
5
|
+
import scala.util.Try
|
6
|
+
import s3.website.model.Encoding._
|
7
|
+
import org.apache.commons.codec.digest.DigestUtils
|
8
|
+
import java.util.zip.GZIPOutputStream
|
9
|
+
import org.apache.commons.io.IOUtils
|
10
|
+
import org.apache.tika.Tika
|
11
|
+
import s3.website.Ruby._
|
12
|
+
import s3.website.model.Encoding.Gzip
|
13
|
+
import scala.util.Failure
|
14
|
+
import scala.Some
|
15
|
+
import scala.util.Success
|
16
|
+
import s3.website.model.Encoding.Zopfli
|
17
|
+
|
18
|
+
object Encoding {
|
19
|
+
|
20
|
+
val defaultGzipExtensions = ".html" :: ".css" :: ".js" :: ".txt" :: Nil
|
21
|
+
|
22
|
+
case class Gzip()
|
23
|
+
case class Zopfli()
|
24
|
+
|
25
|
+
def encodingOnS3(path: String)(implicit site: Site): Option[Either[Gzip, Zopfli]] =
|
26
|
+
site.config.gzip.flatMap { (gzipSetting: Either[Boolean, Seq[String]]) =>
|
27
|
+
val shouldZipThisFile = gzipSetting.fold(
|
28
|
+
shouldGzip => defaultGzipExtensions exists path.endsWith,
|
29
|
+
fileExtensions => fileExtensions exists path.endsWith
|
30
|
+
)
|
31
|
+
if (shouldZipThisFile && site.config.gzip_zopfli.isDefined)
|
32
|
+
Some(Right(Zopfli()))
|
33
|
+
else if (shouldZipThisFile)
|
34
|
+
Some(Left(Gzip()))
|
35
|
+
else
|
36
|
+
None
|
37
|
+
}
|
38
|
+
|
39
|
+
type MD5 = String
|
40
|
+
}
|
41
|
+
|
42
|
+
sealed trait S3KeyProvider {
|
43
|
+
def s3Key: String
|
44
|
+
}
|
45
|
+
|
46
|
+
trait UploadTypeResolved {
|
47
|
+
def uploadType: UploadType
|
48
|
+
}
|
49
|
+
|
50
|
+
sealed trait UploadType // Sealed, so that we can avoid inexhaustive pattern matches more easily
|
51
|
+
|
52
|
+
case object NewFile extends UploadType
|
53
|
+
case object Update extends UploadType
|
54
|
+
|
55
|
+
case class LocalFile(
|
56
|
+
s3Key: String,
|
57
|
+
sourceFile: File,
|
58
|
+
encodingOnS3: Option[Either[Gzip, Zopfli]]
|
59
|
+
) extends S3KeyProvider
|
60
|
+
|
61
|
+
object LocalFile {
|
62
|
+
def toUpload(localFile: LocalFile)(implicit config: Config): Either[Error, Upload] = Try {
|
63
|
+
def fis(file: File): InputStream = new FileInputStream(file)
|
64
|
+
def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
|
65
|
+
val sourceFile: File = localFile
|
66
|
+
.encodingOnS3
|
67
|
+
.fold(localFile.sourceFile)(algorithm => {
|
68
|
+
val tempFile = File.createTempFile(localFile.sourceFile.getName, "gzip")
|
69
|
+
tempFile.deleteOnExit()
|
70
|
+
using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
|
71
|
+
IOUtils.copy(fis(localFile.sourceFile), stream)
|
72
|
+
}
|
73
|
+
tempFile
|
74
|
+
})
|
75
|
+
val md5 = using(fis(sourceFile)) { inputStream =>
|
76
|
+
DigestUtils.md5Hex(inputStream)
|
77
|
+
}
|
78
|
+
val maxAge = config.max_age.flatMap { maxAgeIntOrGlob =>
|
79
|
+
maxAgeIntOrGlob.fold(
|
80
|
+
(seconds: Int) => Some(seconds),
|
81
|
+
(globs2Ints: Map[String, Int]) =>
|
82
|
+
globs2Ints.find { globAndInt =>
|
83
|
+
(rubyRuntime evalScriptlet s"File.fnmatch('${globAndInt._1}', '${localFile.s3Key}')")
|
84
|
+
.toJava(classOf[Boolean])
|
85
|
+
.asInstanceOf[Boolean]
|
86
|
+
} map (_._2)
|
87
|
+
)
|
88
|
+
}
|
89
|
+
|
90
|
+
Upload(
|
91
|
+
s3Key = localFile.s3Key,
|
92
|
+
essence = Right(
|
93
|
+
UploadBody(
|
94
|
+
md5 = md5,
|
95
|
+
contentEncoding = localFile.encodingOnS3.map(_ => "gzip"),
|
96
|
+
contentLength = sourceFile.length(),
|
97
|
+
maxAge = maxAge,
|
98
|
+
contentType = tika.detect(localFile.sourceFile),
|
99
|
+
openInputStream = () => new FileInputStream(sourceFile)
|
100
|
+
)
|
101
|
+
)
|
102
|
+
)
|
103
|
+
} match {
|
104
|
+
case Success(upload) => Right(upload)
|
105
|
+
case Failure(error) => Left(IOError(error))
|
106
|
+
}
|
107
|
+
|
108
|
+
lazy val tika = new Tika()
|
109
|
+
|
110
|
+
def resolveLocalFiles(implicit site: Site): Either[Error, Seq[LocalFile]] = Try {
|
111
|
+
val files = recursiveListFiles(new File(site.rootDirectory)).filterNot(_.isDirectory)
|
112
|
+
files map { file =>
|
113
|
+
val s3Key = site.resolveS3Key(file)
|
114
|
+
LocalFile(s3Key, file, encodingOnS3(s3Key))
|
115
|
+
} filterNot { file =>
|
116
|
+
site.config.exclude_from_upload exists { _.fold(
|
117
|
+
// For backward compatibility, use Ruby regex matching
|
118
|
+
(exclusionRegex: String) => rubyRegexMatches(file.s3Key, exclusionRegex),
|
119
|
+
(exclusionRegexes: Seq[String]) => exclusionRegexes exists (rubyRegexMatches(file.s3Key, _))
|
120
|
+
) }
|
121
|
+
} filterNot { _.sourceFile.getName == "s3_website.yml" } // For security reasons, the s3_website.yml should never be pushed
|
122
|
+
} match {
|
123
|
+
case Success(localFiles) =>
|
124
|
+
Right(
|
125
|
+
// Sort by key, because this will improve the performance when pushing existing sites.
|
126
|
+
// The lazy-loading diff take advantage of this arrangement.
|
127
|
+
localFiles sortBy (_.s3Key)
|
128
|
+
)
|
129
|
+
case Failure(error) =>
|
130
|
+
Left(IOError(error))
|
131
|
+
}
|
132
|
+
|
133
|
+
def recursiveListFiles(f: File): Seq[File] = {
|
134
|
+
val these = f.listFiles
|
135
|
+
these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
|
136
|
+
}
|
137
|
+
}
|
138
|
+
|
139
|
+
case class OverrideExisting()
|
140
|
+
case class CreateNew()
|
141
|
+
|
142
|
+
case class Redirect(key: String, redirectTarget: String)
|
143
|
+
|
144
|
+
object Redirect extends UploadType {
|
145
|
+
def resolveRedirects(implicit config: Config): Seq[Upload with UploadTypeResolved] = {
|
146
|
+
val redirects = config.redirects.fold(Nil: Seq[Redirect]) {
|
147
|
+
sourcesToTargets =>
|
148
|
+
sourcesToTargets.foldLeft(Seq(): Seq[Redirect]) {
|
149
|
+
(redirects, sourceToTarget) =>
|
150
|
+
redirects :+ Redirect(sourceToTarget._1, sourceToTarget._2)
|
151
|
+
}
|
152
|
+
}
|
153
|
+
redirects.map { redirect =>
|
154
|
+
Upload.apply(redirect)
|
155
|
+
}
|
156
|
+
}
|
157
|
+
}
|
158
|
+
|
159
|
+
case class Upload(
|
160
|
+
s3Key: String,
|
161
|
+
essence: Either[Redirect, UploadBody]
|
162
|
+
) extends S3KeyProvider {
|
163
|
+
|
164
|
+
def withUploadType(ut: UploadType) =
|
165
|
+
new Upload(s3Key, essence) with UploadTypeResolved {
|
166
|
+
def uploadType = ut
|
167
|
+
}
|
168
|
+
}
|
169
|
+
|
170
|
+
object Upload {
|
171
|
+
def apply(redirect: Redirect): Upload with UploadTypeResolved = new Upload(redirect.key, Left(redirect)) with UploadTypeResolved {
|
172
|
+
def uploadType = Redirect
|
173
|
+
}
|
174
|
+
}
|
175
|
+
|
176
|
+
/**
|
177
|
+
* Represents a bunch of data that should be stored into an S3 objects body.
|
178
|
+
*/
|
179
|
+
case class UploadBody(
|
180
|
+
md5: MD5,
|
181
|
+
contentLength: Long,
|
182
|
+
contentEncoding: Option[String],
|
183
|
+
maxAge: Option[Int],
|
184
|
+
contentType: String,
|
185
|
+
openInputStream: () => InputStream // It's in the caller's responsibility to close this stream
|
186
|
+
)
|
187
|
+
|
188
|
+
case class S3File(s3Key: String, md5: MD5)
|
189
|
+
|
190
|
+
object S3File {
|
191
|
+
def apply(summary: S3ObjectSummary): S3File = S3File(summary.getKey, summary.getETag)
|
192
|
+
}
|
@@ -0,0 +1,445 @@
|
|
1
|
+
package s3.website
|
2
|
+
|
3
|
+
import org.specs2.mutable.{After, Specification}
|
4
|
+
import s3.website.model._
|
5
|
+
import org.specs2.specification.Scope
|
6
|
+
import org.apache.commons.io.FileUtils
|
7
|
+
import org.apache.commons.codec.digest.DigestUtils
|
8
|
+
import java.io.File
|
9
|
+
import scala.util.Random
|
10
|
+
import org.mockito.{Mockito, Matchers, ArgumentCaptor}
|
11
|
+
import org.mockito.Mockito._
|
12
|
+
import com.amazonaws.services.s3.AmazonS3
|
13
|
+
import com.amazonaws.services.s3.model._
|
14
|
+
import scala.concurrent.ExecutionContext.Implicits.global
|
15
|
+
import scala.concurrent.Await
|
16
|
+
import scala.concurrent.duration._
|
17
|
+
import s3.website.S3.S3ClientProvider
|
18
|
+
import scala.collection.JavaConversions._
|
19
|
+
import s3.website.model.NewFile
|
20
|
+
import scala.Some
|
21
|
+
import com.amazonaws.AmazonServiceException
|
22
|
+
import org.apache.commons.codec.digest.DigestUtils.md5Hex
|
23
|
+
import s3.website.CloudFront.CloudFrontClientProvider
|
24
|
+
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
25
|
+
import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
|
26
|
+
import org.mockito.stubbing.Answer
|
27
|
+
import org.mockito.invocation.InvocationOnMock
|
28
|
+
|
29
|
+
class S3WebsiteSpec extends Specification {
|
30
|
+
|
31
|
+
"gzip: true" should {
|
32
|
+
"update a gzipped S3 object if the contents has changed" in new SiteDirectory with MockAWS {
|
33
|
+
implicit val site = siteWithFilesAndContent(
|
34
|
+
config = defaultConfig.copy(gzip = Some(Left(true))),
|
35
|
+
localFilesWithContent = ("styles.css", "<h1>hi again</h1>") :: Nil
|
36
|
+
)
|
37
|
+
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
38
|
+
Push.pushSite
|
39
|
+
sentPutObjectRequest.getKey must equalTo("styles.css")
|
40
|
+
}
|
41
|
+
|
42
|
+
"not update a gzipped S3 object if the contents has not changed" in new SiteDirectory with MockAWS {
|
43
|
+
implicit val site = siteWithFilesAndContent(
|
44
|
+
config = defaultConfig.copy(gzip = Some(Left(true))),
|
45
|
+
localFilesWithContent = ("styles.css", "<h1>hi</h1>") :: Nil
|
46
|
+
)
|
47
|
+
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
48
|
+
Push.pushSite
|
49
|
+
noUploadsOccurred must beTrue
|
50
|
+
}
|
51
|
+
}
|
52
|
+
|
53
|
+
"""
|
54
|
+
gzip:
|
55
|
+
- .xml
|
56
|
+
""" should {
|
57
|
+
"update a gzipped S3 object if the contents has changed" in new SiteDirectory with MockAWS {
|
58
|
+
implicit val site = siteWithFilesAndContent(
|
59
|
+
config = defaultConfig.copy(gzip = Some(Right(".xml" :: Nil))),
|
60
|
+
localFilesWithContent = ("file.xml", "<h1>hi again</h1>") :: Nil
|
61
|
+
)
|
62
|
+
setS3Files(S3File("file.xml", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
63
|
+
Push.pushSite
|
64
|
+
sentPutObjectRequest.getKey must equalTo("file.xml")
|
65
|
+
}
|
66
|
+
}
|
67
|
+
|
68
|
+
|
69
|
+
"push" should {
|
70
|
+
"not upload a file if it has not changed" in new SiteDirectory with MockAWS {
|
71
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<div>hello</div>") :: Nil)
|
72
|
+
setS3Files(S3File("index.html", md5Hex("<div>hello</div>")))
|
73
|
+
Push.pushSite
|
74
|
+
noUploadsOccurred must beTrue
|
75
|
+
}
|
76
|
+
|
77
|
+
"update a file if it has changed" in new SiteDirectory with MockAWS {
|
78
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>old text</h1>") :: Nil)
|
79
|
+
setS3Files(S3File("index.html", md5Hex("<h1>new text</h1>")))
|
80
|
+
Push.pushSite
|
81
|
+
sentPutObjectRequest.getKey must equalTo("index.html")
|
82
|
+
}
|
83
|
+
|
84
|
+
"create a file if does not exist on S3" in new SiteDirectory with MockAWS {
|
85
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
86
|
+
Push.pushSite
|
87
|
+
sentPutObjectRequest.getKey must equalTo("index.html")
|
88
|
+
}
|
89
|
+
|
90
|
+
"delete files that are on S3 but not on local file system" in new SiteDirectory with MockAWS {
|
91
|
+
implicit val site = buildSite()
|
92
|
+
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
93
|
+
Push.pushSite
|
94
|
+
sentDelete must equalTo("old.html")
|
95
|
+
}
|
96
|
+
|
97
|
+
"invalidate the CloudFront items" in new SiteDirectory with MockAWS {
|
98
|
+
implicit val site = siteWithFiles(
|
99
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
|
100
|
+
localFiles = "test.css" :: "articles/index.html" :: Nil
|
101
|
+
)
|
102
|
+
Push.pushSite
|
103
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/test.css" :: "/articles/index.html" :: Nil).sorted)
|
104
|
+
}
|
105
|
+
|
106
|
+
"not send CloudFront invalidation requests on redirect objects" in new SiteDirectory with MockAWS {
|
107
|
+
implicit val site = buildSite(
|
108
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z"), redirects = Some(Map("/index.php" -> "index.html")))
|
109
|
+
)
|
110
|
+
Push.pushSite
|
111
|
+
noInvalidationsOccurred must beTrue
|
112
|
+
}
|
113
|
+
|
114
|
+
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new SiteDirectory with MockAWS {
|
115
|
+
setTooManyInvalidationsInProgress(4)
|
116
|
+
implicit val site = siteWithFiles(
|
117
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
|
118
|
+
localFiles = "test.css" :: Nil
|
119
|
+
)
|
120
|
+
Push.pushSite must equalTo(0) // The retries should finally result in a success
|
121
|
+
sentInvalidationRequests.length must equalTo(4)
|
122
|
+
}
|
123
|
+
}
|
124
|
+
|
125
|
+
"cloudfront_invalidate_root: true" should {
|
126
|
+
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new SiteDirectory with MockAWS {
|
127
|
+
implicit val site = siteWithFiles(
|
128
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z"), cloudfront_invalidate_root = Some(true)),
|
129
|
+
localFiles = "index.html" :: "articles/index.html" :: Nil
|
130
|
+
)
|
131
|
+
Push.pushSite
|
132
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/" :: "/articles/" :: Nil).sorted)
|
133
|
+
}
|
134
|
+
}
|
135
|
+
|
136
|
+
"a site with over 1000 items" should {
|
137
|
+
"split the CloudFront invalidation requests into batches of 1000 items" in new SiteDirectory with MockAWS {
|
138
|
+
implicit val site = siteWithFiles(
|
139
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
|
140
|
+
localFiles = (1 to 1002).map { i => s"file-$i"}
|
141
|
+
)
|
142
|
+
Push.pushSite
|
143
|
+
sentInvalidationRequests.length must equalTo(2)
|
144
|
+
sentInvalidationRequests(0).getInvalidationBatch.getPaths.getItems.length must equalTo(1000)
|
145
|
+
sentInvalidationRequests(1).getInvalidationBatch.getPaths.getItems.length must equalTo(2)
|
146
|
+
}
|
147
|
+
}
|
148
|
+
|
149
|
+
"push exit status" should {
|
150
|
+
"be 0 all uploads succeed" in new SiteDirectory with MockAWS {
|
151
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
152
|
+
Push.pushSite must equalTo(0)
|
153
|
+
}
|
154
|
+
|
155
|
+
"be 1 if any of the uploads fails" in new SiteDirectory with MockAWS {
|
156
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
157
|
+
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
158
|
+
Push.pushSite must equalTo(1)
|
159
|
+
}
|
160
|
+
|
161
|
+
"be 0 if CloudFront invalidations and uploads succeed"in new SiteDirectory with MockAWS {
|
162
|
+
implicit val site = siteWithFiles(
|
163
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
|
164
|
+
localFiles = "test.css" :: "articles/index.html" :: Nil
|
165
|
+
)
|
166
|
+
Push.pushSite must equalTo(0)
|
167
|
+
}
|
168
|
+
|
169
|
+
"be 1 if CloudFront invalidation fails"in new SiteDirectory with MockAWS {
|
170
|
+
setCloudFrontAsInternallyBroken()
|
171
|
+
implicit val site = siteWithFiles(
|
172
|
+
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
|
173
|
+
localFiles = "test.css" :: "articles/index.html" :: Nil
|
174
|
+
)
|
175
|
+
Push.pushSite must equalTo(1)
|
176
|
+
}
|
177
|
+
}
|
178
|
+
|
179
|
+
"s3_website.yml file" should {
|
180
|
+
"never be uploaded" in new SiteDirectory with MockAWS {
|
181
|
+
implicit val site = siteWithFiles(localFiles = "s3_website.yml" :: Nil)
|
182
|
+
Push.pushSite
|
183
|
+
noUploadsOccurred must beTrue
|
184
|
+
}
|
185
|
+
}
|
186
|
+
|
187
|
+
"exclude_from_upload: string" should {
|
188
|
+
"result in matching files not being uploaded" in new SiteDirectory with MockAWS {
|
189
|
+
implicit val site = siteWithFiles(
|
190
|
+
config = defaultConfig.copy(exclude_from_upload = Some(Left(".DS_.*?"))),
|
191
|
+
localFiles = ".DS_Store" :: Nil
|
192
|
+
)
|
193
|
+
Push.pushSite
|
194
|
+
noUploadsOccurred must beTrue
|
195
|
+
}
|
196
|
+
}
|
197
|
+
|
198
|
+
"""
|
199
|
+
exclude_from_upload:
|
200
|
+
- regex
|
201
|
+
- another_exclusion
|
202
|
+
""" should {
|
203
|
+
"result in matching files not being uploaded" in new SiteDirectory with MockAWS {
|
204
|
+
implicit val site = siteWithFiles(
|
205
|
+
config = defaultConfig.copy(exclude_from_upload = Some(Right(".DS_.*?" :: "logs" :: Nil))),
|
206
|
+
localFiles = ".DS_Store" :: "logs/test.log" :: Nil
|
207
|
+
)
|
208
|
+
Push.pushSite
|
209
|
+
noUploadsOccurred must beTrue
|
210
|
+
}
|
211
|
+
}
|
212
|
+
|
213
|
+
"ignore_on_server: value" should {
|
214
|
+
"not delete the S3 objects that match the ignore value" in new SiteDirectory with MockAWS {
|
215
|
+
implicit val site = buildSite(config = defaultConfig.copy(ignore_on_server = Some(Left("logs"))))
|
216
|
+
setS3Files(S3File("logs/log.txt", ""))
|
217
|
+
Push.pushSite
|
218
|
+
noDeletesOccurred must beTrue
|
219
|
+
}
|
220
|
+
}
|
221
|
+
|
222
|
+
"""
|
223
|
+
ignore_on_server:
|
224
|
+
- regex
|
225
|
+
- another_ignore
|
226
|
+
""" should {
|
227
|
+
"not delete the S3 objects that match the ignore value" in new SiteDirectory with MockAWS {
|
228
|
+
implicit val site = buildSite(config = defaultConfig.copy(ignore_on_server = Some(Right(".*txt" :: Nil))))
|
229
|
+
setS3Files(S3File("logs/log.txt", ""))
|
230
|
+
Push.pushSite
|
231
|
+
noDeletesOccurred must beTrue
|
232
|
+
}
|
233
|
+
}
|
234
|
+
|
235
|
+
"max-age in config" can {
|
236
|
+
"be applied to all files" in new SiteDirectory with MockAWS {
|
237
|
+
implicit val site = siteWithFiles(defaultConfig.copy(max_age = Some(Left(60))), localFiles = "index.html" :: Nil)
|
238
|
+
Push.pushSite
|
239
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
240
|
+
}
|
241
|
+
|
242
|
+
"be applied to files that match the glob" in new SiteDirectory with MockAWS {
|
243
|
+
implicit val site = siteWithFiles(defaultConfig.copy(max_age = Some(Right(Map("*.html" -> 90)))), localFiles = "index.html" :: Nil)
|
244
|
+
Push.pushSite
|
245
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
246
|
+
}
|
247
|
+
|
248
|
+
"be applied to directories that match the glob" in new SiteDirectory with MockAWS {
|
249
|
+
implicit val site = siteWithFiles(defaultConfig.copy(max_age = Some(Right(Map("assets/**/*.js" -> 90)))), localFiles = "assets/lib/jquery.js" :: Nil)
|
250
|
+
Push.pushSite
|
251
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
252
|
+
}
|
253
|
+
|
254
|
+
"not be applied if the glob doesn't match" in new SiteDirectory with MockAWS {
|
255
|
+
implicit val site = siteWithFiles(defaultConfig.copy(max_age = Some(Right(Map("*.js" -> 90)))), localFiles = "index.html" :: Nil)
|
256
|
+
Push.pushSite
|
257
|
+
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
258
|
+
}
|
259
|
+
}
|
260
|
+
|
261
|
+
"s3_reduced_redundancy: true in config" should {
|
262
|
+
"result in uploads being marked with reduced redundancy" in new SiteDirectory with MockAWS {
|
263
|
+
implicit val site = siteWithFiles(defaultConfig.copy(s3_reduced_redundancy = Some(true)), localFiles = "index.html" :: Nil)
|
264
|
+
Push.pushSite
|
265
|
+
sentPutObjectRequest.getStorageClass must equalTo("REDUCED_REDUNDANCY")
|
266
|
+
}
|
267
|
+
}
|
268
|
+
|
269
|
+
"s3_reduced_redundancy: false in config" should {
|
270
|
+
"result in uploads being marked with the default storage class" in new SiteDirectory with MockAWS {
|
271
|
+
implicit val site = siteWithFiles(defaultConfig.copy(s3_reduced_redundancy = Some(false)), localFiles = "index.html" :: Nil)
|
272
|
+
Push.pushSite
|
273
|
+
sentPutObjectRequest.getStorageClass must beNull
|
274
|
+
}
|
275
|
+
}
|
276
|
+
|
277
|
+
"redirect in config" should {
|
278
|
+
"result in a redirect instruction that is sent to AWS" in new SiteDirectory with MockAWS {
|
279
|
+
implicit val site = buildSite(defaultConfig.copy(redirects = Some(Map("index.php" -> "/index.html"))))
|
280
|
+
Push.pushSite
|
281
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
282
|
+
}
|
283
|
+
|
284
|
+
"result in max-age=0 Cache-Control header on the object" in new SiteDirectory with MockAWS {
|
285
|
+
implicit val site = buildSite(defaultConfig.copy(redirects = Some(Map("index.php" -> "/index.html"))))
|
286
|
+
Push.pushSite
|
287
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=0, no-cache")
|
288
|
+
}
|
289
|
+
}
|
290
|
+
|
291
|
+
"redirect in config and an object on the S3 bucket" should {
|
292
|
+
"not result in the S3 object being deleted" in new SiteDirectory with MockAWS {
|
293
|
+
implicit val site = siteWithFiles(
|
294
|
+
localFiles = "index.html" :: Nil,
|
295
|
+
config = defaultConfig.copy(redirects = Some(Map("index.php" -> "/index.html")))
|
296
|
+
)
|
297
|
+
setS3Files(S3File("index.php", "md5"))
|
298
|
+
Push.pushSite
|
299
|
+
noDeletesOccurred must beTrue
|
300
|
+
}
|
301
|
+
}
|
302
|
+
|
303
|
+
"dotfiles" should {
|
304
|
+
"be included in the pushed files" in new SiteDirectory with MockAWS {
|
305
|
+
implicit val site = siteWithFiles(localFiles = ".vimrc" :: Nil)
|
306
|
+
Push.pushSite
|
307
|
+
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
308
|
+
}
|
309
|
+
}
|
310
|
+
|
311
|
+
trait MockAWS extends MockS3 with MockCloudFront with Scope
|
312
|
+
|
313
|
+
trait MockCloudFront {
|
314
|
+
val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
|
315
|
+
implicit val cfClientProvider: CloudFrontClientProvider = _ => amazonCloudFrontClient
|
316
|
+
implicit val cloudFrontSleepTimeUnit: TimeUnit = MILLISECONDS
|
317
|
+
|
318
|
+
def sentInvalidationRequests: Seq[CreateInvalidationRequest] = {
|
319
|
+
val createInvalidationReq = ArgumentCaptor.forClass(classOf[CreateInvalidationRequest])
|
320
|
+
verify(amazonCloudFrontClient, Mockito.atLeastOnce()).createInvalidation(createInvalidationReq.capture())
|
321
|
+
createInvalidationReq.getAllValues
|
322
|
+
}
|
323
|
+
|
324
|
+
def sentInvalidationRequest = sentInvalidationRequests.ensuring(_.length == 1).head
|
325
|
+
|
326
|
+
def noInvalidationsOccurred = {
|
327
|
+
verify(amazonCloudFrontClient, Mockito.never()).createInvalidation(Matchers.any(classOf[CreateInvalidationRequest]))
|
328
|
+
true // Mockito is based on exceptions
|
329
|
+
}
|
330
|
+
|
331
|
+
def setTooManyInvalidationsInProgress(attemptWhenInvalidationSucceeds: Int) {
|
332
|
+
var callCount = 0
|
333
|
+
doAnswer(new Answer[CreateInvalidationResult] {
|
334
|
+
override def answer(invocation: InvocationOnMock): CreateInvalidationResult = {
|
335
|
+
callCount += 1
|
336
|
+
if (callCount < attemptWhenInvalidationSucceeds)
|
337
|
+
throw new TooManyInvalidationsInProgressException("just too many, man")
|
338
|
+
else
|
339
|
+
mock(classOf[CreateInvalidationResult])
|
340
|
+
}
|
341
|
+
}).when(amazonCloudFrontClient).createInvalidation(Matchers.anyObject())
|
342
|
+
}
|
343
|
+
|
344
|
+
def setCloudFrontAsInternallyBroken() {
|
345
|
+
when(amazonCloudFrontClient.createInvalidation(Matchers.anyObject())).thenThrow(new AmazonServiceException("CloudFront is down"))
|
346
|
+
}
|
347
|
+
}
|
348
|
+
|
349
|
+
trait MockS3 {
|
350
|
+
val amazonS3Client = mock(classOf[AmazonS3])
|
351
|
+
implicit val s3ClientProvider: S3ClientProvider = _ => amazonS3Client
|
352
|
+
val s3ObjectListing = new ObjectListing
|
353
|
+
when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
|
354
|
+
|
355
|
+
def setS3Files(s3Files: S3File*) {
|
356
|
+
s3Files.foreach { s3File =>
|
357
|
+
s3ObjectListing.getObjectSummaries.add({
|
358
|
+
val summary = new S3ObjectSummary
|
359
|
+
summary.setETag(s3File.md5)
|
360
|
+
summary.setKey(s3File.s3Key)
|
361
|
+
summary
|
362
|
+
})
|
363
|
+
}
|
364
|
+
}
|
365
|
+
|
366
|
+
val s3 = new S3()
|
367
|
+
|
368
|
+
def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
|
369
|
+
Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
|
370
|
+
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
371
|
+
verify(amazonS3Client).putObject(req.capture())
|
372
|
+
req.getValue
|
373
|
+
}
|
374
|
+
|
375
|
+
def sentPutObjectRequests: Seq[PutObjectRequest] = {
|
376
|
+
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
377
|
+
verify(amazonS3Client).putObject(req.capture())
|
378
|
+
req.getAllValues
|
379
|
+
}
|
380
|
+
|
381
|
+
def sentPutObjectRequest = sentPutObjectRequests.ensuring(_.length == 1).head
|
382
|
+
|
383
|
+
def sentDeletes: Seq[S3Key] = {
|
384
|
+
val deleteKey = ArgumentCaptor.forClass(classOf[S3Key])
|
385
|
+
verify(amazonS3Client).deleteObject(Matchers.anyString(), deleteKey.capture())
|
386
|
+
deleteKey.getAllValues
|
387
|
+
}
|
388
|
+
|
389
|
+
def sentDelete = sentDeletes.ensuring(_.length == 1).head
|
390
|
+
|
391
|
+
def noDeletesOccurred = {
|
392
|
+
verify(amazonS3Client, never()).deleteObject(Matchers.anyString(), Matchers.anyString())
|
393
|
+
true // Mockito is based on exceptions
|
394
|
+
}
|
395
|
+
|
396
|
+
def noUploadsOccurred = {
|
397
|
+
verify(amazonS3Client, never()).putObject(Matchers.any(classOf[PutObjectRequest]))
|
398
|
+
true // Mockito is based on exceptions
|
399
|
+
}
|
400
|
+
|
401
|
+
type S3Key = String
|
402
|
+
}
|
403
|
+
|
404
|
+
trait SiteDirectory extends After {
|
405
|
+
val siteDir = new File(FileUtils.getTempDirectory, "site" + Random.nextLong())
|
406
|
+
siteDir.mkdir()
|
407
|
+
|
408
|
+
def after {
|
409
|
+
FileUtils.forceDelete(siteDir)
|
410
|
+
}
|
411
|
+
|
412
|
+
def buildSite(config: Config = defaultConfig): Site = Site(siteDir.getAbsolutePath, config)
|
413
|
+
|
414
|
+
def siteWithFilesAndContent(config: Config = defaultConfig, localFilesWithContent: Seq[(String, String)]): Site = {
|
415
|
+
localFilesWithContent.foreach {
|
416
|
+
case (filePath, content) =>
|
417
|
+
val file = new File(siteDir, filePath)
|
418
|
+
FileUtils.forceMkdir(file.getParentFile)
|
419
|
+
file.createNewFile()
|
420
|
+
FileUtils.write(file, content)
|
421
|
+
}
|
422
|
+
buildSite(config)
|
423
|
+
}
|
424
|
+
|
425
|
+
def siteWithFiles(config: Config = defaultConfig, localFiles: Seq[String]): Site =
|
426
|
+
siteWithFilesAndContent(config, localFilesWithContent = localFiles.map((_, "file contents")))
|
427
|
+
}
|
428
|
+
|
429
|
+
val defaultConfig = Config(
|
430
|
+
s3_id = "foo",
|
431
|
+
s3_secret = "bar",
|
432
|
+
s3_bucket = "bucket",
|
433
|
+
s3_endpoint = S3Endpoint.defaultEndpoint,
|
434
|
+
max_age = None,
|
435
|
+
gzip = None,
|
436
|
+
gzip_zopfli = None,
|
437
|
+
ignore_on_server = None,
|
438
|
+
exclude_from_upload = None,
|
439
|
+
s3_reduced_redundancy = None,
|
440
|
+
cloudfront_distribution_id = None,
|
441
|
+
cloudfront_invalidate_root = None,
|
442
|
+
redirects = None,
|
443
|
+
concurrency_level = 1
|
444
|
+
)
|
445
|
+
}
|