s3_website_revived 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +15 -0
- data/.travis.yml +5 -0
- data/Gemfile +3 -0
- data/LICENSE +42 -0
- data/README.md +591 -0
- data/Rakefile +2 -0
- data/additional-docs/debugging.md +21 -0
- data/additional-docs/development.md +29 -0
- data/additional-docs/example-configurations.md +113 -0
- data/additional-docs/running-from-ec2-with-dropbox.md +6 -0
- data/additional-docs/setting-up-aws-credentials.md +52 -0
- data/assembly.sbt +3 -0
- data/bin/s3_website +285 -0
- data/build.sbt +48 -0
- data/changelog.md +596 -0
- data/lib/s3_website/version.rb +3 -0
- data/lib/s3_website.rb +7 -0
- data/project/assembly.sbt +1 -0
- data/project/build.properties +1 -0
- data/project/plugins.sbt +1 -0
- data/release +41 -0
- data/resources/configuration_file_template.yml +67 -0
- data/resources/s3_website.jar.md5 +1 -0
- data/s3_website-4.0.0.jar +0 -0
- data/s3_website.gemspec +34 -0
- data/sbt +3 -0
- data/src/main/resources/log4j.properties +6 -0
- data/src/main/scala/s3/website/ByteHelper.scala +18 -0
- data/src/main/scala/s3/website/CloudFront.scala +144 -0
- data/src/main/scala/s3/website/Logger.scala +67 -0
- data/src/main/scala/s3/website/Push.scala +246 -0
- data/src/main/scala/s3/website/Ruby.scala +14 -0
- data/src/main/scala/s3/website/S3.scala +239 -0
- data/src/main/scala/s3/website/UploadHelper.scala +76 -0
- data/src/main/scala/s3/website/model/Config.scala +249 -0
- data/src/main/scala/s3/website/model/S3Endpoint.scala +35 -0
- data/src/main/scala/s3/website/model/Site.scala +159 -0
- data/src/main/scala/s3/website/model/push.scala +225 -0
- data/src/main/scala/s3/website/model/ssg.scala +30 -0
- data/src/main/scala/s3/website/package.scala +182 -0
- data/src/test/scala/s3/website/AwsSdkSpec.scala +15 -0
- data/src/test/scala/s3/website/ConfigSpec.scala +150 -0
- data/src/test/scala/s3/website/S3EndpointSpec.scala +15 -0
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +1480 -0
- data/src/test/scala/s3/website/UnitTest.scala +11 -0
- data/vagrant/Vagrantfile +25 -0
- metadata +195 -0
|
@@ -0,0 +1,1480 @@
|
|
|
1
|
+
package s3.website
|
|
2
|
+
|
|
3
|
+
import java.io._
|
|
4
|
+
import java.nio.charset.StandardCharsets
|
|
5
|
+
import java.util.concurrent.atomic.AtomicInteger
|
|
6
|
+
import java.util.zip.{GZIPInputStream, GZIPOutputStream}
|
|
7
|
+
|
|
8
|
+
import com.amazonaws.AmazonServiceException
|
|
9
|
+
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
|
10
|
+
import com.amazonaws.services.cloudfront.model.{CreateInvalidationRequest, CreateInvalidationResult, TooManyInvalidationsInProgressException}
|
|
11
|
+
import com.amazonaws.services.s3.AmazonS3
|
|
12
|
+
import com.amazonaws.services.s3.model._
|
|
13
|
+
import org.apache.commons.codec.digest.DigestUtils._
|
|
14
|
+
import org.apache.commons.io.FileUtils._
|
|
15
|
+
import org.apache.commons.io.IOUtils.{write => _}
|
|
16
|
+
import org.apache.commons.io.{FileUtils, IOUtils}
|
|
17
|
+
import org.mockito.Mockito._
|
|
18
|
+
import org.mockito.invocation.InvocationOnMock
|
|
19
|
+
import org.mockito.stubbing.Answer
|
|
20
|
+
import org.mockito.{ArgumentCaptor, Matchers, Mockito}
|
|
21
|
+
import org.specs2.mutable.{BeforeAfter, Specification}
|
|
22
|
+
import org.specs2.specification.Scope
|
|
23
|
+
import s3.website.CloudFront.CloudFrontSetting
|
|
24
|
+
import s3.website.Push.CliArgs
|
|
25
|
+
import s3.website.S3.S3Setting
|
|
26
|
+
import s3.website.UploadHelper.DELETE_NOTHING_MAGIC_WORD
|
|
27
|
+
import s3.website.model.Config.S3_website_yml
|
|
28
|
+
import s3.website.model.Ssg.automaticallySupportedSiteGenerators
|
|
29
|
+
import s3.website.model._
|
|
30
|
+
|
|
31
|
+
import scala.collection.JavaConversions._
|
|
32
|
+
import scala.collection.mutable
|
|
33
|
+
import scala.concurrent.duration._
|
|
34
|
+
import scala.util.Random
|
|
35
|
+
|
|
36
|
+
class S3WebsiteSpec extends Specification {
|
|
37
|
+
|
|
38
|
+
"gzip: true" should {
|
|
39
|
+
"update a gzipped S3 object if the contents has changed" in new BasicSetup {
|
|
40
|
+
config = "gzip: true"
|
|
41
|
+
setLocalFileWithContent(("styles.css", "<h1>hi again</h1>"))
|
|
42
|
+
setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
|
|
43
|
+
push()
|
|
44
|
+
sentPutObjectRequest.getKey must equalTo("styles.css")
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
"gzip a file" in new BasicSetup {
|
|
48
|
+
val htmlString = "<h1>hi again</h1>"
|
|
49
|
+
config = "gzip: true"
|
|
50
|
+
setLocalFileWithContent(("index.html", htmlString))
|
|
51
|
+
setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
|
|
52
|
+
val putObjectRequestCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
|
53
|
+
push()
|
|
54
|
+
sentPutObjectRequest.getKey must equalTo("index.html")
|
|
55
|
+
verify(amazonS3Client).putObject(putObjectRequestCaptor.capture())
|
|
56
|
+
|
|
57
|
+
val bytesToS3: InputStream = putObjectRequestCaptor.getValue.getInputStream
|
|
58
|
+
val unzippedBytesToS3 = new GZIPInputStream(bytesToS3)
|
|
59
|
+
val unzippedString = IOUtils.toString(unzippedBytesToS3, StandardCharsets.UTF_8)
|
|
60
|
+
|
|
61
|
+
unzippedString must equalTo(htmlString)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
"apply the correct mime type on an already gzipped file" in new BasicSetup {
|
|
65
|
+
val htmlString = "<html><body><h1>Hello world</h1></body></html>"
|
|
66
|
+
val gzippedHtml = gzip(htmlString.getBytes(StandardCharsets.UTF_8))
|
|
67
|
+
config = "gzip: true"
|
|
68
|
+
setLocalFileWithContent("index.html", gzippedHtml)
|
|
69
|
+
push()
|
|
70
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
"apply the correct mime type on an already gzipped .json file" in new BasicSetup {
|
|
74
|
+
val jsonString = """" {"json": true} """
|
|
75
|
+
val gzippedJson = gzip(jsonString.getBytes(StandardCharsets.UTF_8))
|
|
76
|
+
config = """
|
|
77
|
+
|gzip:
|
|
78
|
+
| - .json
|
|
79
|
+
"""
|
|
80
|
+
setLocalFileWithContent("test.json", gzippedJson)
|
|
81
|
+
push()
|
|
82
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("application/json; charset=utf-8")
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
"not gzip the file if it's already gzipped" in new BasicSetup {
|
|
86
|
+
config = "gzip: true"
|
|
87
|
+
|
|
88
|
+
val cssString = "body { color: red }"
|
|
89
|
+
val gzippedCss = gzip(cssString.getBytes(StandardCharsets.UTF_8))
|
|
90
|
+
setLocalFileWithContent("styles.css", gzippedCss)
|
|
91
|
+
val putObjectRequestCaptor = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
|
92
|
+
push()
|
|
93
|
+
sentPutObjectRequest.getKey must equalTo("styles.css")
|
|
94
|
+
verify(amazonS3Client).putObject(putObjectRequestCaptor.capture())
|
|
95
|
+
|
|
96
|
+
val bytesToS3: InputStream = putObjectRequestCaptor.getValue.getInputStream
|
|
97
|
+
val unzippedBytesToS3 = new GZIPInputStream(bytesToS3)
|
|
98
|
+
val unzippedString = IOUtils.toString(unzippedBytesToS3, StandardCharsets.UTF_8)
|
|
99
|
+
|
|
100
|
+
unzippedString must equalTo(cssString)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
def gzip(data: Array[Byte]): Array[Byte] = {
|
|
104
|
+
def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
|
|
105
|
+
|
|
106
|
+
val gzippedOutputStream: ByteArrayOutputStream = new ByteArrayOutputStream
|
|
107
|
+
using(new GZIPOutputStream(gzippedOutputStream)) { stream =>
|
|
108
|
+
IOUtils.copy(new ByteArrayInputStream(data), stream)
|
|
109
|
+
}
|
|
110
|
+
gzippedOutputStream.toByteArray
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
"not update a gzipped S3 object if the contents has not changed" in new BasicSetup {
|
|
114
|
+
config = "gzip: true"
|
|
115
|
+
setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
|
|
116
|
+
setS3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
|
|
117
|
+
push()
|
|
118
|
+
noUploadsOccurred must beTrue
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
(".html" :: ".css" :: ".js" :: ".txt" :: ".ico" :: Nil).foreach((fileExtension) =>
|
|
122
|
+
s"gzip files that end with $fileExtension" in new BasicSetup {
|
|
123
|
+
config = "gzip: true"
|
|
124
|
+
setLocalFileWithContent((s"file.$fileExtension", "file contents"))
|
|
125
|
+
push()
|
|
126
|
+
sentPutObjectRequest.getMetadata.getContentEncoding must equalTo("gzip")
|
|
127
|
+
}
|
|
128
|
+
)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
"""
|
|
132
|
+
gzip:
|
|
133
|
+
- .xml
|
|
134
|
+
""" should {
|
|
135
|
+
"update a gzipped S3 object if the contents has changed" in new BasicSetup {
|
|
136
|
+
config = """
|
|
137
|
+
|gzip:
|
|
138
|
+
| - .xml
|
|
139
|
+
""".stripMargin
|
|
140
|
+
setLocalFileWithContent(("file.xml", "<h1>hi again</h1>"))
|
|
141
|
+
setS3File("file.xml", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
|
|
142
|
+
push()
|
|
143
|
+
sentPutObjectRequest.getKey must equalTo("file.xml")
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
"push" should {
|
|
148
|
+
"not upload a file if it has not changed" in new BasicSetup {
|
|
149
|
+
setLocalFileWithContent(("index.html", "<div>hello</div>"))
|
|
150
|
+
setS3File("index.html", md5Hex("<div>hello</div>"))
|
|
151
|
+
push()
|
|
152
|
+
noUploadsOccurred must beTrue
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
"not upload a file if it has not changed and s3_key_prefix is defined" in new BasicSetup {
|
|
156
|
+
config = "s3_key_prefix: test"
|
|
157
|
+
setLocalFileWithContent(("index.html", "<div>hello</div>"))
|
|
158
|
+
setS3File("test/index.html", md5Hex("<div>hello</div>"))
|
|
159
|
+
push()
|
|
160
|
+
noUploadsOccurred must beTrue
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
"detect a changed file even though another file has the same contents as the changed file" in new BasicSetup {
|
|
164
|
+
setLocalFilesWithContent(("1.txt", "foo"), ("2.txt", "foo"))
|
|
165
|
+
setS3File("1.txt", md5Hex("bar"))
|
|
166
|
+
setS3File("2.txt", md5Hex("foo"))
|
|
167
|
+
push()
|
|
168
|
+
sentPutObjectRequest.getKey must equalTo("1.txt")
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
"update a file if it has changed" in new BasicSetup {
|
|
172
|
+
setLocalFileWithContent(("index.html", "<h1>old text</h1>"))
|
|
173
|
+
setS3File("index.html", md5Hex("<h1>new text</h1>"))
|
|
174
|
+
push()
|
|
175
|
+
sentPutObjectRequest.getKey must equalTo("index.html")
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
"create a file if does not exist on S3" in new BasicSetup {
|
|
179
|
+
setLocalFile("index.html")
|
|
180
|
+
push()
|
|
181
|
+
sentPutObjectRequest.getKey must equalTo("index.html")
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
"delete files that are on S3 but not on local file system" in new BasicSetup {
|
|
185
|
+
setS3File("old.html", md5Hex("<h1>old text</h1>"))
|
|
186
|
+
push()
|
|
187
|
+
sentDelete must equalTo("old.html")
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
"delete files that match the s3_key_prefix" in new BasicSetup {
|
|
191
|
+
config = "s3_key_prefix: production"
|
|
192
|
+
setS3File("production/old.html", md5Hex("<h1>old text</h1>"))
|
|
193
|
+
push()
|
|
194
|
+
sentDelete must equalTo("production/old.html")
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
"retain files that do not match the s3_key_prefix" in new BasicSetup {
|
|
198
|
+
config = "s3_key_prefix: production"
|
|
199
|
+
setS3File("old.html", md5Hex("<h1>old text</h1>"))
|
|
200
|
+
push()
|
|
201
|
+
noDeletesOccurred
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
"retain files that do not match the s3_key_prefix" in new BasicSetup {
|
|
205
|
+
config = "s3_key_prefix: test"
|
|
206
|
+
setS3File("test1.html")
|
|
207
|
+
push()
|
|
208
|
+
noDeletesOccurred
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
"try again if the upload fails" in new BasicSetup {
|
|
212
|
+
setLocalFile("index.html")
|
|
213
|
+
uploadFailsAndThenSucceeds(howManyFailures = 5)
|
|
214
|
+
push()
|
|
215
|
+
verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
"not try again if the upload fails on because of invalid credentials" in new BasicSetup {
|
|
219
|
+
setLocalFile("index.html")
|
|
220
|
+
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
|
|
221
|
+
val e = new AmazonServiceException("your credentials are incorrect")
|
|
222
|
+
e.setStatusCode(403)
|
|
223
|
+
e
|
|
224
|
+
}
|
|
225
|
+
push()
|
|
226
|
+
verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
"try again if the request times out" in new BasicSetup {
|
|
230
|
+
var attempt = 0
|
|
231
|
+
when(amazonS3Client putObject Matchers.any(classOf[PutObjectRequest])) thenAnswer new Answer[PutObjectResult] {
|
|
232
|
+
def answer(invocation: InvocationOnMock) = {
|
|
233
|
+
attempt += 1
|
|
234
|
+
if (attempt < 2) {
|
|
235
|
+
val e = new AmazonServiceException("Too long a request")
|
|
236
|
+
e.setStatusCode(400)
|
|
237
|
+
e.setErrorCode("RequestTimeout")
|
|
238
|
+
throw e
|
|
239
|
+
} else {
|
|
240
|
+
new PutObjectResult
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
setLocalFile("index.html")
|
|
245
|
+
val exitStatus = push()
|
|
246
|
+
verify(amazonS3Client, times(2)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
"try again if the delete fails" in new BasicSetup {
|
|
250
|
+
setS3File("old.html", md5Hex("<h1>old text</h1>"))
|
|
251
|
+
deleteFailsAndThenSucceeds(howManyFailures = 5)
|
|
252
|
+
push()
|
|
253
|
+
verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
"try again if the object listing fails" in new BasicSetup {
|
|
257
|
+
setS3File("old.html", md5Hex("<h1>old text</h1>"))
|
|
258
|
+
objectListingFailsAndThenSucceeds(howManyFailures = 5)
|
|
259
|
+
push()
|
|
260
|
+
verify(amazonS3Client, times(6)).listObjects(Matchers.any(classOf[ListObjectsRequest]))
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
"push with CloudFront" should {
|
|
265
|
+
"invalidate the updated CloudFront items" in new BasicSetup {
|
|
266
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
267
|
+
setLocalFiles("css/test.css", "articles/index.html")
|
|
268
|
+
setOutdatedS3Keys("css/test.css", "articles/index.html")
|
|
269
|
+
push()
|
|
270
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/css/test.css" :: "/articles/index.html" :: Nil).sorted)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
"not send CloudFront invalidation requests on new objects" in new BasicSetup {
|
|
274
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
275
|
+
setLocalFile("newfile.js")
|
|
276
|
+
push()
|
|
277
|
+
noInvalidationsOccurred must beTrue
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
"not send CloudFront invalidation requests on redirect objects" in new BasicSetup {
|
|
281
|
+
config = """
|
|
282
|
+
|cloudfront_distribution_id: EGM1J2JJX9Z
|
|
283
|
+
|redirects:
|
|
284
|
+
| /index.php: index.html
|
|
285
|
+
""".stripMargin
|
|
286
|
+
push()
|
|
287
|
+
noInvalidationsOccurred must beTrue
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new BasicSetup {
|
|
291
|
+
setTooManyInvalidationsInProgress(4)
|
|
292
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
293
|
+
setLocalFile("test.css")
|
|
294
|
+
setOutdatedS3Keys("test.css")
|
|
295
|
+
push() must equalTo(0) // The retries should finally result in a success
|
|
296
|
+
sentInvalidationRequests.length must equalTo(4)
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
"retry if CloudFront is temporarily unreachable" in new BasicSetup {
|
|
300
|
+
invalidationsFailAndThenSucceed(5)
|
|
301
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
302
|
+
setLocalFile("test.css")
|
|
303
|
+
setOutdatedS3Keys("test.css")
|
|
304
|
+
push()
|
|
305
|
+
sentInvalidationRequests.length must equalTo(6)
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
"encode unsafe characters in the keys" in new BasicSetup {
|
|
309
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
310
|
+
setLocalFile("articles/arnold's file.html")
|
|
311
|
+
setOutdatedS3Keys("articles/arnold's file.html")
|
|
312
|
+
push()
|
|
313
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/arnold%27s%20file.html" :: Nil).sorted)
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
"encode unsafe characters in the keys" in new BasicSetup {
|
|
317
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
318
|
+
setLocalFile("articles/Äöüßñé.html")
|
|
319
|
+
setOutdatedS3Keys("articles/Äöüßñé.html")
|
|
320
|
+
push()
|
|
321
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/%C3%84%C3%B6%C3%BC%C3%9F%C3%B1%C3%A9.html" :: Nil).sorted)
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
"invalidate the root object '/' if a top-level object is updated or deleted" in new BasicSetup {
|
|
325
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
326
|
+
setLocalFile("maybe-index.html")
|
|
327
|
+
setOutdatedS3Keys("maybe-index.html")
|
|
328
|
+
push()
|
|
329
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/" :: "/maybe-index.html" :: Nil).sorted)
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
333
|
+
config = """
|
|
334
|
+
|cloudfront_distribution_id: EGM1J2JJX9Z
|
|
335
|
+
|s3_key_prefix: production
|
|
336
|
+
""".stripMargin
|
|
337
|
+
setLocalFile("index.html")
|
|
338
|
+
setOutdatedS3Keys("production/index.html")
|
|
339
|
+
push()
|
|
340
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/production/index.html" :: Nil).sorted)
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
"cloudfront_invalidate_root: true" should {
|
|
345
|
+
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new BasicSetup {
|
|
346
|
+
config = """
|
|
347
|
+
|cloudfront_distribution_id: EGM1J2JJX9Z
|
|
348
|
+
|cloudfront_invalidate_root: true
|
|
349
|
+
""".stripMargin
|
|
350
|
+
setLocalFile("articles/index.html")
|
|
351
|
+
setOutdatedS3Keys("articles/index.html")
|
|
352
|
+
push()
|
|
353
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq must contain("/articles/")
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
"always invalidate the root path" in new BasicSetup {
|
|
357
|
+
config = """
|
|
358
|
+
|cloudfront_distribution_id: EGM1J2JJX9Z
|
|
359
|
+
|cloudfront_invalidate_root: true
|
|
360
|
+
""".stripMargin
|
|
361
|
+
setLocalFile("articles/index.html")
|
|
362
|
+
setOutdatedS3Keys("articles/index.html")
|
|
363
|
+
push()
|
|
364
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq must contain("/index.html")
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
"treat the s3_key_prefix as the root path" in new BasicSetup {
|
|
368
|
+
config = """
|
|
369
|
+
|cloudfront_distribution_id: EGM1J2JJX9Z
|
|
370
|
+
|cloudfront_invalidate_root: true
|
|
371
|
+
|s3_key_prefix: test
|
|
372
|
+
""".stripMargin
|
|
373
|
+
setLocalFile("articles/index.html")
|
|
374
|
+
setOutdatedS3Keys("test/articles/index.html")
|
|
375
|
+
push()
|
|
376
|
+
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(
|
|
377
|
+
("/test/index.html" :: "/test/articles/" :: Nil).sorted
|
|
378
|
+
)
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
"not invalidate anything if there was nothing to push" in new BasicSetup {
|
|
382
|
+
config = """
|
|
383
|
+
|cloudfront_distribution_id: EGM1J2JJX9Z
|
|
384
|
+
|cloudfront_invalidate_root: true
|
|
385
|
+
""".stripMargin
|
|
386
|
+
push()
|
|
387
|
+
noInvalidationsOccurred
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
"a site with over 1000 items" should {
|
|
392
|
+
"split the CloudFront invalidation requests into batches of 1000 items" in new BasicSetup {
|
|
393
|
+
val files = (1 to 1002).map { i => s"lots-of-files/file-$i"}
|
|
394
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
395
|
+
setLocalFiles(files:_*)
|
|
396
|
+
setOutdatedS3Keys(files:_*)
|
|
397
|
+
push()
|
|
398
|
+
sentInvalidationRequests.length must equalTo(2)
|
|
399
|
+
sentInvalidationRequests(0).getInvalidationBatch.getPaths.getItems.length must equalTo(1000)
|
|
400
|
+
sentInvalidationRequests(1).getInvalidationBatch.getPaths.getItems.length must equalTo(2)
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
"push exit status" should {
|
|
405
|
+
"be 0 all uploads succeed" in new BasicSetup {
|
|
406
|
+
setLocalFiles("file.txt")
|
|
407
|
+
push() must equalTo(0)
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
"be 1 if any of the uploads fails" in new BasicSetup {
|
|
411
|
+
setLocalFiles("file.txt")
|
|
412
|
+
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
|
413
|
+
push() must equalTo(1)
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
"be 1 if any of the redirects fails" in new BasicSetup {
|
|
417
|
+
config = """
|
|
418
|
+
|redirects:
|
|
419
|
+
| index.php: /index.html
|
|
420
|
+
""".stripMargin
|
|
421
|
+
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
|
422
|
+
push() must equalTo(1)
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
"be 0 if CloudFront invalidations and uploads succeed"in new BasicSetup {
|
|
426
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
427
|
+
setLocalFile("test.css")
|
|
428
|
+
setOutdatedS3Keys("test.css")
|
|
429
|
+
push() must equalTo(0)
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
"be 1 if CloudFront is unreachable or broken"in new BasicSetup {
|
|
433
|
+
setCloudFrontAsInternallyBroken()
|
|
434
|
+
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
|
435
|
+
setLocalFile("test.css")
|
|
436
|
+
setOutdatedS3Keys("test.css")
|
|
437
|
+
push() must equalTo(1)
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
"be 0 if upload retry succeeds" in new BasicSetup {
|
|
441
|
+
setLocalFile("index.html")
|
|
442
|
+
uploadFailsAndThenSucceeds(howManyFailures = 1)
|
|
443
|
+
push() must equalTo(0)
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
"be 1 if delete retry fails" in new BasicSetup {
|
|
447
|
+
setLocalFile("index.html")
|
|
448
|
+
uploadFailsAndThenSucceeds(howManyFailures = 6)
|
|
449
|
+
push() must equalTo(1)
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
"be 1 if an object listing fails" in new BasicSetup {
|
|
453
|
+
setS3File("old.html", md5Hex("<h1>old text</h1>"))
|
|
454
|
+
objectListingFailsAndThenSucceeds(howManyFailures = 6)
|
|
455
|
+
push() must equalTo(1)
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
"s3_key_prefix in config" should {
|
|
460
|
+
"apply the prefix into all the S3 keys" in new BasicSetup {
|
|
461
|
+
config = "s3_key_prefix: production"
|
|
462
|
+
setLocalFile("index.html")
|
|
463
|
+
push()
|
|
464
|
+
sentPutObjectRequest.getKey must equalTo("production/index.html")
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
"work with slash" in new BasicSetup {
|
|
468
|
+
config = "s3_key_prefix: production/"
|
|
469
|
+
setLocalFile("index.html")
|
|
470
|
+
push()
|
|
471
|
+
sentPutObjectRequest.getKey must equalTo("production/index.html")
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
"s3_website.yml file" should {
|
|
476
|
+
"never be uploaded" in new BasicSetup {
|
|
477
|
+
setLocalFile("s3_website.yml")
|
|
478
|
+
push()
|
|
479
|
+
noUploadsOccurred must beTrue
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
"never be uploaded even when s3_key_prefix is defined" in new BasicSetup {
|
|
483
|
+
config = "s3_key_prefix: production"
|
|
484
|
+
setLocalFile("s3_website.yml")
|
|
485
|
+
push()
|
|
486
|
+
noUploadsOccurred must beTrue
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
".env file" should { // The .env file is the https://github.com/bkeepers/dotenv file
|
|
491
|
+
"never be uploaded" in new BasicSetup {
|
|
492
|
+
setLocalFile(".env")
|
|
493
|
+
push()
|
|
494
|
+
noUploadsOccurred must beTrue
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
"never be uploaded even when s3_key_prefix is defined" in new BasicSetup {
|
|
498
|
+
config = "s3_key_prefix: production"
|
|
499
|
+
setLocalFile(".env")
|
|
500
|
+
push()
|
|
501
|
+
noUploadsOccurred must beTrue
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
"exclude_from_upload: string" should {
|
|
506
|
+
"result in matching files not being uploaded" in new BasicSetup {
|
|
507
|
+
config = "exclude_from_upload: .DS_.*?"
|
|
508
|
+
setLocalFile(".DS_Store")
|
|
509
|
+
push()
|
|
510
|
+
noUploadsOccurred must beTrue
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
514
|
+
config = """
|
|
515
|
+
|s3_key_prefix: production
|
|
516
|
+
|exclude_from_upload: hello.txt
|
|
517
|
+
""".stripMargin
|
|
518
|
+
setLocalFile("hello.txt")
|
|
519
|
+
push()
|
|
520
|
+
noUploadsOccurred must beTrue
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
"""
|
|
525
|
+
exclude_from_upload:
|
|
526
|
+
- regex
|
|
527
|
+
- another_exclusion
|
|
528
|
+
""" should {
|
|
529
|
+
"result in matching files not being uploaded" in new BasicSetup {
|
|
530
|
+
config = """
|
|
531
|
+
|exclude_from_upload:
|
|
532
|
+
| - .DS_.*?
|
|
533
|
+
| - logs
|
|
534
|
+
""".stripMargin
|
|
535
|
+
setLocalFiles(".DS_Store", "logs/test.log")
|
|
536
|
+
push()
|
|
537
|
+
noUploadsOccurred must beTrue
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
541
|
+
config = """
|
|
542
|
+
|s3_key_prefix: production
|
|
543
|
+
|exclude_from_upload:
|
|
544
|
+
|- hello.txt
|
|
545
|
+
""".stripMargin
|
|
546
|
+
setLocalFile("hello.txt")
|
|
547
|
+
push()
|
|
548
|
+
noUploadsOccurred must beTrue
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
"ignore_on_server: value" should {
|
|
553
|
+
"not delete the S3 objects that match the ignore value" in new BasicSetup {
|
|
554
|
+
config = "ignore_on_server: logs"
|
|
555
|
+
setS3File("logs/log.txt")
|
|
556
|
+
push()
|
|
557
|
+
noDeletesOccurred must beTrue
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
"support non-US-ASCII files" in new BasicSetup {
|
|
561
|
+
setS3File("tags/笔记/test.html", "")
|
|
562
|
+
config = "ignore_on_server: tags/笔记/test.html"
|
|
563
|
+
push()
|
|
564
|
+
noDeletesOccurred must beTrue
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
568
|
+
config = """
|
|
569
|
+
|s3_key_prefix: production
|
|
570
|
+
|ignore_on_server: hello.txt
|
|
571
|
+
""".stripMargin
|
|
572
|
+
setS3File("hello.txt")
|
|
573
|
+
push()
|
|
574
|
+
noDeletesOccurred must beTrue
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
"ignore_on_server: _DELETE_NOTHING_ON_THE_S3_BUCKET_" should {
|
|
579
|
+
"result in no files being deleted on the S3 bucket" in new BasicSetup {
|
|
580
|
+
config = s"""
|
|
581
|
+
|ignore_on_server: $DELETE_NOTHING_MAGIC_WORD
|
|
582
|
+
""".stripMargin
|
|
583
|
+
setS3File("file.txt")
|
|
584
|
+
push()
|
|
585
|
+
noDeletesOccurred
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
589
|
+
config = s"""
|
|
590
|
+
|s3_key_prefix: production
|
|
591
|
+
|ignore_on_server: $DELETE_NOTHING_MAGIC_WORD
|
|
592
|
+
""".stripMargin
|
|
593
|
+
setS3File("file.txt")
|
|
594
|
+
push()
|
|
595
|
+
noDeletesOccurred
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
"""
|
|
600
|
+
ignore_on_server:
|
|
601
|
+
- regex
|
|
602
|
+
- another_ignore
|
|
603
|
+
""" should {
|
|
604
|
+
"not delete the S3 objects that match the ignore value" in new BasicSetup {
|
|
605
|
+
config = """
|
|
606
|
+
|ignore_on_server:
|
|
607
|
+
| - .*txt
|
|
608
|
+
""".stripMargin
|
|
609
|
+
setS3File("logs/log.txt", "")
|
|
610
|
+
push()
|
|
611
|
+
noDeletesOccurred must beTrue
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
"support non-US-ASCII files" in new BasicSetup {
|
|
615
|
+
setS3File("tags/笔记/test.html", "")
|
|
616
|
+
config = """
|
|
617
|
+
|ignore_on_server:
|
|
618
|
+
| - tags/笔记/test.html
|
|
619
|
+
""".stripMargin
|
|
620
|
+
push()
|
|
621
|
+
noDeletesOccurred must beTrue
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
625
|
+
config = """
|
|
626
|
+
|s3_key_prefix: production
|
|
627
|
+
|ignore_on_server:
|
|
628
|
+
|- hello.*
|
|
629
|
+
""".stripMargin
|
|
630
|
+
setS3File("hello.txt")
|
|
631
|
+
push()
|
|
632
|
+
noDeletesOccurred must beTrue
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
"error message" should {
|
|
637
|
+
"be helpful when the site setting in s3_website.yml points to a non-existing file" in new BasicSetup {
|
|
638
|
+
config = "site: nonexisting_site_dir"
|
|
639
|
+
val logEntries = new mutable.MutableList[String]
|
|
640
|
+
push(logCapturer = Some((logEntry: String) =>
|
|
641
|
+
logEntries += logEntry
|
|
642
|
+
))
|
|
643
|
+
logEntries must contain(
|
|
644
|
+
s"Could not find a website. (The site setting in s3_website.yml points to a non-existing file $siteDirectory/nonexisting_site_dir)"
|
|
645
|
+
)
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
"site in config" should {
|
|
650
|
+
"let the user deploy a site from a custom location" in new CustomSiteDirectory with EmptySite with MockAWS with DefaultRunMode {
|
|
651
|
+
config = s"site: $siteDirectory"
|
|
652
|
+
setLocalFile(".vimrc")
|
|
653
|
+
|
|
654
|
+
new File(siteDirectory, ".vimrc").exists() must beTrue // Sanity check
|
|
655
|
+
siteDirectory must not equalTo workingDirectory // Sanity check
|
|
656
|
+
|
|
657
|
+
push()
|
|
658
|
+
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
"not override the --site command-line switch" in new BasicSetup {
|
|
662
|
+
config = s"site: ${System.getProperty("java.io.tmpdir")}"
|
|
663
|
+
setLocalFile(".vimrc") // This creates a file in the directory into which the --site CLI arg points
|
|
664
|
+
push()
|
|
665
|
+
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
automaticallySupportedSiteGenerators foreach { siteGenerator =>
|
|
669
|
+
"override an automatically detected site" in new CustomSiteDirectory with EmptySite with MockAWS with DefaultRunMode {
|
|
670
|
+
addContentToAutomaticallyDetectedSite(workingDirectory)
|
|
671
|
+
config = s"site: $siteDirectory"
|
|
672
|
+
setLocalFile(".vimrc") // Add content to the custom site directory
|
|
673
|
+
push()
|
|
674
|
+
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
def addContentToAutomaticallyDetectedSite(workingDirectory: File) {
|
|
678
|
+
val automaticallyDetectedSiteDir = new File(workingDirectory, siteGenerator.outputDirectory)
|
|
679
|
+
automaticallyDetectedSiteDir.mkdirs()
|
|
680
|
+
write(new File(automaticallyDetectedSiteDir, ".bashrc"), "echo hello")
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
"cache_control in config" should {
|
|
686
|
+
"be applied to all files" in new BasicSetup {
|
|
687
|
+
config = "cache_control: public, no-transform, max-age=1200, s-maxage=1200"
|
|
688
|
+
setLocalFile("index.html")
|
|
689
|
+
push()
|
|
690
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, no-transform, max-age=1200, s-maxage=1200")
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
694
|
+
config =
|
|
695
|
+
"""
|
|
696
|
+
|cache_control: public, no-transform, max-age=1200, s-maxage=1200
|
|
697
|
+
|s3_key_prefix: foo
|
|
698
|
+
""".stripMargin
|
|
699
|
+
setLocalFile("index.html")
|
|
700
|
+
push()
|
|
701
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, no-transform, max-age=1200, s-maxage=1200")
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
"should take precedence over max_age" in new BasicSetup {
|
|
705
|
+
config = """
|
|
706
|
+
|max_age: 120
|
|
707
|
+
|cache_control: public, max-age=90
|
|
708
|
+
""".stripMargin
|
|
709
|
+
setLocalFile("index.html")
|
|
710
|
+
push()
|
|
711
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, max-age=90")
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
"log a warning if both cache_control and max_age are present" in new BasicSetup {
|
|
715
|
+
val logEntries = new mutable.MutableList[String]
|
|
716
|
+
config = """
|
|
717
|
+
|max_age: 120
|
|
718
|
+
|cache_control: public, max-age=90
|
|
719
|
+
""".stripMargin
|
|
720
|
+
setLocalFile("index.html")
|
|
721
|
+
push(logCapturer = Some((logEntry: String) =>
|
|
722
|
+
logEntries += logEntry
|
|
723
|
+
))
|
|
724
|
+
logEntries must contain("[\u001B[33mwarn\u001B[0m] Overriding the max_age setting with the cache_control settin")
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
"supports all valid URI characters in the glob setting" in new BasicSetup {
|
|
728
|
+
config = """
|
|
729
|
+
|cache_control:
|
|
730
|
+
| "*.html": public, max-age=120
|
|
731
|
+
""".stripMargin
|
|
732
|
+
val allValidUrlCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~:/?#[]@!$&'()*+,;=" // See http://stackoverflow.com/a/1547940/990356 for discussion
|
|
733
|
+
setLocalFile(s"$allValidUrlCharacters.html")
|
|
734
|
+
push()
|
|
735
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, max-age=120")
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
"be applied to files that match the glob" in new BasicSetup {
|
|
739
|
+
config = """
|
|
740
|
+
|cache_control:
|
|
741
|
+
| "*.html": no-store
|
|
742
|
+
""".stripMargin
|
|
743
|
+
setLocalFile("index.html")
|
|
744
|
+
push()
|
|
745
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("no-store")
|
|
746
|
+
}
|
|
747
|
+
|
|
748
|
+
"be applied to directories that match the glob" in new BasicSetup {
|
|
749
|
+
config = """
|
|
750
|
+
|cache_control:
|
|
751
|
+
| "assets/**/*.js": no-cache, no-store
|
|
752
|
+
""".stripMargin
|
|
753
|
+
setLocalFile("assets/lib/jquery.js")
|
|
754
|
+
push()
|
|
755
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("no-cache, no-store")
|
|
756
|
+
}
|
|
757
|
+
|
|
758
|
+
"not be applied if the glob doesn't match" in new BasicSetup {
|
|
759
|
+
config = """
|
|
760
|
+
|cache_control:
|
|
761
|
+
| "*.js": max-age=120
|
|
762
|
+
""".stripMargin
|
|
763
|
+
setLocalFile("index.html")
|
|
764
|
+
push()
|
|
765
|
+
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
"support non-US-ASCII directory names" in new BasicSetup {
|
|
769
|
+
config = """
|
|
770
|
+
|cache_control:
|
|
771
|
+
| "*": no-cache
|
|
772
|
+
""".stripMargin
|
|
773
|
+
setLocalFile("tags/笔记/index.html")
|
|
774
|
+
push() must equalTo(0)
|
|
775
|
+
}
|
|
776
|
+
|
|
777
|
+
"have overlapping definitions in the glob, and then the most specific glob will win" in new BasicSetup {
|
|
778
|
+
config = """
|
|
779
|
+
|cache_control:
|
|
780
|
+
| "*.js": no-cache, no-store
|
|
781
|
+
| "assets/**/*.js": public, must-revalidate, max-age=120
|
|
782
|
+
""".stripMargin
|
|
783
|
+
setLocalFile("assets/lib/jquery.js")
|
|
784
|
+
push()
|
|
785
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, must-revalidate, max-age=120")
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
789
|
+
config =
|
|
790
|
+
"""
|
|
791
|
+
|cache_control:
|
|
792
|
+
| "*.html": public, no-transform, max-age=1200, s-maxage=1200
|
|
793
|
+
|s3_key_prefix: foo
|
|
794
|
+
""".stripMargin
|
|
795
|
+
setLocalFile("index.html")
|
|
796
|
+
push()
|
|
797
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("public, no-transform, max-age=1200, s-maxage=1200")
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
"cache control" can {
|
|
802
|
+
"be undefined" in new BasicSetup {
|
|
803
|
+
setLocalFile("index.html")
|
|
804
|
+
push()
|
|
805
|
+
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
"max_age in config" can {
|
|
810
|
+
"be applied to all files" in new BasicSetup {
|
|
811
|
+
config = "max_age: 60"
|
|
812
|
+
setLocalFile("index.html")
|
|
813
|
+
push()
|
|
814
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
818
|
+
config =
|
|
819
|
+
"""
|
|
820
|
+
|max_age: 60
|
|
821
|
+
|s3_key_prefix: test
|
|
822
|
+
""".stripMargin
|
|
823
|
+
setLocalFile("index.html")
|
|
824
|
+
push()
|
|
825
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
"supports all valid URI characters in the glob setting" in new BasicSetup {
|
|
829
|
+
config = """
|
|
830
|
+
|max_age:
|
|
831
|
+
| "*.html": 90
|
|
832
|
+
""".stripMargin
|
|
833
|
+
val allValidUrlCharacters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~:/?#[]@!$&'()*+,;=" // See http://stackoverflow.com/a/1547940/990356 for discussion
|
|
834
|
+
setLocalFile(s"$allValidUrlCharacters.html")
|
|
835
|
+
push()
|
|
836
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
"be applied to files that match the glob" in new BasicSetup {
|
|
840
|
+
config = """
|
|
841
|
+
|max_age:
|
|
842
|
+
| "*.html": 90
|
|
843
|
+
""".stripMargin
|
|
844
|
+
setLocalFile("index.html")
|
|
845
|
+
push()
|
|
846
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
"be applied to directories that match the glob" in new BasicSetup {
|
|
850
|
+
config = """
|
|
851
|
+
|max_age:
|
|
852
|
+
| "assets/**/*.js": 90
|
|
853
|
+
""".stripMargin
|
|
854
|
+
setLocalFile("assets/lib/jquery.js")
|
|
855
|
+
push()
|
|
856
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
"not be applied if the glob doesn't match" in new BasicSetup {
|
|
860
|
+
config = """
|
|
861
|
+
|max_age:
|
|
862
|
+
| "*.js": 90
|
|
863
|
+
""".stripMargin
|
|
864
|
+
setLocalFile("index.html")
|
|
865
|
+
push()
|
|
866
|
+
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
"be used to disable caching" in new BasicSetup {
|
|
870
|
+
config = "max_age: 0"
|
|
871
|
+
setLocalFile("index.html")
|
|
872
|
+
push()
|
|
873
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("no-cache, max-age=0")
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
"support non-US-ASCII directory names" in new BasicSetup {
|
|
877
|
+
config = """
|
|
878
|
+
|max_age:
|
|
879
|
+
| "*": 21600
|
|
880
|
+
""".stripMargin
|
|
881
|
+
setLocalFile("tags/笔记/index.html")
|
|
882
|
+
push() must equalTo(0)
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
"have overlapping definitions in the glob, and then the most specific glob will win" in new BasicSetup {
|
|
886
|
+
config = """
|
|
887
|
+
|max_age:
|
|
888
|
+
| "*.js": 33
|
|
889
|
+
| "assets/**/*.js": 90
|
|
890
|
+
""".stripMargin
|
|
891
|
+
setLocalFile("assets/lib/jquery.js")
|
|
892
|
+
push()
|
|
893
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
"respect the more specific glob" in new BasicSetup {
|
|
897
|
+
config = """
|
|
898
|
+
|max_age:
|
|
899
|
+
| "assets/*": 150
|
|
900
|
+
| "assets/*.gif": 86400
|
|
901
|
+
""".stripMargin
|
|
902
|
+
setLocalFiles("assets/jquery.js", "assets/picture.gif")
|
|
903
|
+
push()
|
|
904
|
+
sentPutObjectRequests.find(_.getKey == "assets/jquery.js").get.getMetadata.getCacheControl must equalTo("max-age=150")
|
|
905
|
+
sentPutObjectRequests.find(_.getKey == "assets/picture.gif").get.getMetadata.getCacheControl must equalTo("max-age=86400")
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
"work with s3_key_prefix" in new BasicSetup {
|
|
909
|
+
config =
|
|
910
|
+
"""
|
|
911
|
+
|max_age:
|
|
912
|
+
| "*.html": 60
|
|
913
|
+
|s3_key_prefix: test
|
|
914
|
+
""".stripMargin
|
|
915
|
+
setLocalFile("index.html")
|
|
916
|
+
push()
|
|
917
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
"s3_reduced_redundancy: true in config" should {
|
|
922
|
+
"result in uploads being marked with reduced redundancy" in new BasicSetup {
|
|
923
|
+
config = "s3_reduced_redundancy: true"
|
|
924
|
+
setLocalFile("file.exe")
|
|
925
|
+
push()
|
|
926
|
+
sentPutObjectRequest.getStorageClass must equalTo("REDUCED_REDUNDANCY")
|
|
927
|
+
}
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
"s3_reduced_redundancy: false in config" should {
|
|
931
|
+
"result in uploads being marked with the default storage class" in new BasicSetup {
|
|
932
|
+
config = "s3_reduced_redundancy: false"
|
|
933
|
+
setLocalFile("file.exe")
|
|
934
|
+
push()
|
|
935
|
+
sentPutObjectRequest.getStorageClass must beNull
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
|
|
939
|
+
"redirect in config" should {
|
|
940
|
+
"result in a redirect instruction that is sent to AWS" in new BasicSetup {
|
|
941
|
+
config = """
|
|
942
|
+
|redirects:
|
|
943
|
+
| index.php: /index.html
|
|
944
|
+
""".stripMargin
|
|
945
|
+
push()
|
|
946
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
"refer to site root when the s3_key_prefix is defined and the redirect target starts with a slash" in new BasicSetup {
|
|
950
|
+
config = """
|
|
951
|
+
|s3_key_prefix: production
|
|
952
|
+
|redirects:
|
|
953
|
+
| index.php: /index.html
|
|
954
|
+
""".stripMargin
|
|
955
|
+
push()
|
|
956
|
+
sentPutObjectRequest.getKey must equalTo("production/index.php")
|
|
957
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
"use s3_key_prefix as the root when the redirect target does not start with a slash" in new BasicSetup {
|
|
961
|
+
config = """
|
|
962
|
+
|s3_key_prefix: production
|
|
963
|
+
|redirects:
|
|
964
|
+
| index.php: index.html
|
|
965
|
+
""".stripMargin
|
|
966
|
+
push()
|
|
967
|
+
sentPutObjectRequest.getKey must equalTo("production/index.php")
|
|
968
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/production/index.html")
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
"add slash to the redirect target" in new BasicSetup {
|
|
972
|
+
config = """
|
|
973
|
+
|redirects:
|
|
974
|
+
| index.php: index.html
|
|
975
|
+
""".stripMargin
|
|
976
|
+
push()
|
|
977
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
"support external redirects" in new BasicSetup {
|
|
981
|
+
config = """
|
|
982
|
+
|redirects:
|
|
983
|
+
| index.php: http://www.youtube.com/watch?v=dQw4w9WgXcQ
|
|
984
|
+
""".stripMargin
|
|
985
|
+
push()
|
|
986
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("http://www.youtube.com/watch?v=dQw4w9WgXcQ")
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
"support external redirects that point to an HTTPS target" in new BasicSetup {
|
|
990
|
+
config = """
|
|
991
|
+
|redirects:
|
|
992
|
+
| index.php: https://www.youtube.com/watch?v=dQw4w9WgXcQ
|
|
993
|
+
""".stripMargin
|
|
994
|
+
push()
|
|
995
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("https://www.youtube.com/watch?v=dQw4w9WgXcQ")
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
"result in max-age=0 Cache-Control header on the object" in new BasicSetup {
|
|
999
|
+
config = """
|
|
1000
|
+
|redirects:
|
|
1001
|
+
| index.php: /index.html
|
|
1002
|
+
""".stripMargin
|
|
1003
|
+
push()
|
|
1004
|
+
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=0, no-cache")
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
"redirect in config and an object on the S3 bucket" should {
|
|
1009
|
+
"not result in the S3 object being deleted" in new BasicSetup {
|
|
1010
|
+
config = """
|
|
1011
|
+
|redirects:
|
|
1012
|
+
| index.php: /index.html
|
|
1013
|
+
""".stripMargin
|
|
1014
|
+
setLocalFile("index.php")
|
|
1015
|
+
setS3File("index.php", "md5")
|
|
1016
|
+
push()
|
|
1017
|
+
noDeletesOccurred must beTrue
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
|
|
1021
|
+
"dotfiles" should {
|
|
1022
|
+
"be included in the pushed files" in new BasicSetup {
|
|
1023
|
+
setLocalFile(".vimrc")
|
|
1024
|
+
push()
|
|
1025
|
+
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
"content type inference" should {
|
|
1030
|
+
"add charset=utf-8 to all html documents" in new BasicSetup {
|
|
1031
|
+
setLocalFile("index.html")
|
|
1032
|
+
push()
|
|
1033
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
"add charset=utf-8 to all text documents" in new BasicSetup {
|
|
1037
|
+
setLocalFile("index.txt")
|
|
1038
|
+
push()
|
|
1039
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/plain; charset=utf-8")
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
"add charset=utf-8 to all json documents" in new BasicSetup {
|
|
1043
|
+
setLocalFile("data.json")
|
|
1044
|
+
push()
|
|
1045
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("application/json; charset=utf-8")
|
|
1046
|
+
}
|
|
1047
|
+
|
|
1048
|
+
"resolve the content type from file contents" in new BasicSetup {
|
|
1049
|
+
setLocalFileWithContent(("index", "<html><body><h1>hi</h1></body></html>"))
|
|
1050
|
+
push()
|
|
1051
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
|
|
1055
|
+
"content_type in config file" should {
|
|
1056
|
+
"override tika's opinion" in new BasicSetup {
|
|
1057
|
+
config = """
|
|
1058
|
+
|content_type:
|
|
1059
|
+
| "*.html": text/foobar
|
|
1060
|
+
""".stripMargin
|
|
1061
|
+
setLocalFileWithContent(("index.html", "<html><body><h1>hi</h1></body></html>"))
|
|
1062
|
+
push()
|
|
1063
|
+
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/foobar; charset=utf-8")
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
|
|
1067
|
+
"ERB in config file" should {
|
|
1068
|
+
"be evaluated" in new BasicSetup {
|
|
1069
|
+
config = """
|
|
1070
|
+
|redirects:
|
|
1071
|
+
|<%= ('a'..'f').to_a.map do |t| ' '+t+ ': /'+t+'.html' end.join('\n')%>
|
|
1072
|
+
""".stripMargin
|
|
1073
|
+
push()
|
|
1074
|
+
sentPutObjectRequests.length must equalTo(6)
|
|
1075
|
+
sentPutObjectRequests.forall(_.getRedirectLocation != null) must beTrue
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
|
|
1079
|
+
"push --force" should {
|
|
1080
|
+
"push all the files whether they have changed or not" in new ForcePush {
|
|
1081
|
+
setLocalFileWithContent(("index.html", "<h1>hi</h1>"))
|
|
1082
|
+
setS3File("index.html", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */)
|
|
1083
|
+
push()
|
|
1084
|
+
sentPutObjectRequest.getKey must equalTo("index.html")
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
"dry run" should {
|
|
1089
|
+
"not push updates" in new DryRun {
|
|
1090
|
+
setLocalFileWithContent(("index.html", "<div>new</div>"))
|
|
1091
|
+
setS3File("index.html", md5Hex("<div>old</div>"))
|
|
1092
|
+
push()
|
|
1093
|
+
noUploadsOccurred must beTrue
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
"not push redirects" in new DryRun {
|
|
1097
|
+
config =
|
|
1098
|
+
"""
|
|
1099
|
+
|redirects:
|
|
1100
|
+
| index.php: /index.html
|
|
1101
|
+
""".stripMargin
|
|
1102
|
+
push()
|
|
1103
|
+
noUploadsOccurred must beTrue
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
"not push deletes" in new DryRun {
|
|
1107
|
+
setS3File("index.html", md5Hex("<div>old</div>"))
|
|
1108
|
+
push()
|
|
1109
|
+
noUploadsOccurred must beTrue
|
|
1110
|
+
}
|
|
1111
|
+
|
|
1112
|
+
"not push new files" in new DryRun {
|
|
1113
|
+
setLocalFile("index.html")
|
|
1114
|
+
push()
|
|
1115
|
+
noUploadsOccurred must beTrue
|
|
1116
|
+
}
|
|
1117
|
+
|
|
1118
|
+
"not invalidate files" in new DryRun {
|
|
1119
|
+
config = "cloudfront_invalidation_id: AABBCC"
|
|
1120
|
+
setS3File("index.html", md5Hex("<div>old</div>"))
|
|
1121
|
+
push()
|
|
1122
|
+
noInvalidationsOccurred must beTrue
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
|
|
1126
|
+
"Jekyll site" should {
|
|
1127
|
+
"be detected automatically" in new JekyllSite with EmptySite with MockAWS with DefaultRunMode {
|
|
1128
|
+
setLocalFile("index.html")
|
|
1129
|
+
push()
|
|
1130
|
+
sentPutObjectRequests.length must equalTo(1)
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
"Nanoc site" should {
|
|
1135
|
+
"be detected automatically" in new NanocSite with EmptySite with MockAWS with DefaultRunMode {
|
|
1136
|
+
setLocalFile("index.html")
|
|
1137
|
+
push()
|
|
1138
|
+
sentPutObjectRequests.length must equalTo(1)
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
"Middleman site" should {
|
|
1143
|
+
"be detected automatically" in new MiddlemanSite with EmptySite with MockAWS with DefaultRunMode {
|
|
1144
|
+
setLocalFile("index.html")
|
|
1145
|
+
push()
|
|
1146
|
+
sentPutObjectRequests.length must equalTo(1)
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
"the setting treat_zero_length_objects_as_redirects" should {
|
|
1151
|
+
"skip application of redirect on a zero-length S3 object" in new BasicSetup {
|
|
1152
|
+
config =
|
|
1153
|
+
"""
|
|
1154
|
+
|treat_zero_length_objects_as_redirects: true
|
|
1155
|
+
|redirects:
|
|
1156
|
+
| index.php: /index.html
|
|
1157
|
+
""".stripMargin
|
|
1158
|
+
setRedirectObject("index.php")
|
|
1159
|
+
push()
|
|
1160
|
+
noUploadsOccurred
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
"not delete the redirect objects on the bucket" in new BasicSetup {
|
|
1164
|
+
config =
|
|
1165
|
+
"""
|
|
1166
|
+
|treat_zero_length_objects_as_redirects: true
|
|
1167
|
+
|redirects:
|
|
1168
|
+
| index.php: /index.html
|
|
1169
|
+
""".stripMargin
|
|
1170
|
+
setRedirectObject("index.php")
|
|
1171
|
+
push()
|
|
1172
|
+
noDeletesOccurred
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
"apply redirects that are missing from S3" in new BasicSetup {
|
|
1176
|
+
config =
|
|
1177
|
+
"""
|
|
1178
|
+
|treat_zero_length_objects_as_redirects: true
|
|
1179
|
+
|redirects:
|
|
1180
|
+
| index.php: /index.html
|
|
1181
|
+
""".stripMargin
|
|
1182
|
+
push()
|
|
1183
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
|
1184
|
+
}
|
|
1185
|
+
|
|
1186
|
+
"apply all redirects when the user invokes `push --force`" in new ForcePush {
|
|
1187
|
+
config =
|
|
1188
|
+
"""
|
|
1189
|
+
|treat_zero_length_objects_as_redirects: true
|
|
1190
|
+
|redirects:
|
|
1191
|
+
| index.php: /index.html
|
|
1192
|
+
""".stripMargin
|
|
1193
|
+
setRedirectObject("index.php")
|
|
1194
|
+
push()
|
|
1195
|
+
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
|
|
1199
|
+
trait BasicSetup extends SiteLocationFromCliArg with EmptySite with MockAWS with DefaultRunMode
|
|
1200
|
+
|
|
1201
|
+
trait ForcePush extends SiteLocationFromCliArg with EmptySite with MockAWS with ForcePushMode
|
|
1202
|
+
|
|
1203
|
+
trait DryRun extends SiteLocationFromCliArg with EmptySite with MockAWS with DryRunMode
|
|
1204
|
+
|
|
1205
|
+
trait DefaultRunMode {
|
|
1206
|
+
implicit def pushOptions: PushOptions = new PushOptions {
|
|
1207
|
+
def dryRun = false
|
|
1208
|
+
def force = false
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
trait DryRunMode {
|
|
1213
|
+
implicit def pushOptions: PushOptions = new PushOptions {
|
|
1214
|
+
def dryRun = true
|
|
1215
|
+
def force = false
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
trait ForcePushMode {
|
|
1220
|
+
implicit def pushOptions: PushOptions = new PushOptions {
|
|
1221
|
+
def dryRun = false
|
|
1222
|
+
def force = true
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
|
|
1226
|
+
|
|
1227
|
+
trait MockAWS extends MockS3 with MockCloudFront with Scope
|
|
1228
|
+
|
|
1229
|
+
trait MockCloudFront extends MockAWSHelper {
|
|
1230
|
+
val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
|
|
1231
|
+
implicit val cfSettings: CloudFrontSetting = CloudFrontSetting(
|
|
1232
|
+
cfClient = _ => amazonCloudFrontClient,
|
|
1233
|
+
retryTimeUnit = MICROSECONDS
|
|
1234
|
+
)
|
|
1235
|
+
|
|
1236
|
+
def sentInvalidationRequests: Seq[CreateInvalidationRequest] = {
|
|
1237
|
+
val createInvalidationReq = ArgumentCaptor.forClass(classOf[CreateInvalidationRequest])
|
|
1238
|
+
verify(amazonCloudFrontClient, Mockito.atLeastOnce()).createInvalidation(createInvalidationReq.capture())
|
|
1239
|
+
createInvalidationReq.getAllValues
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
def sentInvalidationRequest = sentInvalidationRequests.ensuring(_.length == 1).head
|
|
1243
|
+
|
|
1244
|
+
def noInvalidationsOccurred = {
|
|
1245
|
+
verify(amazonCloudFrontClient, Mockito.never()).createInvalidation(Matchers.any(classOf[CreateInvalidationRequest]))
|
|
1246
|
+
true // Mockito is based on exceptions
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
def invalidationsFailAndThenSucceed(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
|
1250
|
+
doAnswer(temporaryFailure(classOf[CreateInvalidationResult]))
|
|
1251
|
+
.when(amazonCloudFrontClient)
|
|
1252
|
+
.createInvalidation(Matchers.anyObject())
|
|
1253
|
+
}
|
|
1254
|
+
|
|
1255
|
+
def setTooManyInvalidationsInProgress(attemptWhenInvalidationSucceeds: Int) {
|
|
1256
|
+
var callCount = 0
|
|
1257
|
+
doAnswer(new Answer[CreateInvalidationResult] {
|
|
1258
|
+
override def answer(invocation: InvocationOnMock): CreateInvalidationResult = {
|
|
1259
|
+
callCount += 1
|
|
1260
|
+
if (callCount < attemptWhenInvalidationSucceeds)
|
|
1261
|
+
throw new TooManyInvalidationsInProgressException("just too many, man")
|
|
1262
|
+
else
|
|
1263
|
+
mock(classOf[CreateInvalidationResult])
|
|
1264
|
+
}
|
|
1265
|
+
}).when(amazonCloudFrontClient).createInvalidation(Matchers.anyObject())
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
def setCloudFrontAsInternallyBroken() {
|
|
1269
|
+
when(amazonCloudFrontClient.createInvalidation(Matchers.anyObject())).thenThrow(new AmazonServiceException("CloudFront is down"))
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
|
|
1273
|
+
trait MockS3 extends MockAWSHelper {
|
|
1274
|
+
val amazonS3Client = mock(classOf[AmazonS3])
|
|
1275
|
+
implicit val s3Settings: S3Setting = S3Setting(
|
|
1276
|
+
s3Client = _ => amazonS3Client,
|
|
1277
|
+
retryTimeUnit = MICROSECONDS
|
|
1278
|
+
)
|
|
1279
|
+
val s3ObjectListing = new ObjectListing
|
|
1280
|
+
when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
|
|
1281
|
+
|
|
1282
|
+
// Simulate the situation where the file on S3 is outdated (as compared to the local file)
|
|
1283
|
+
def setOutdatedS3Keys(s3Keys: String*) {
|
|
1284
|
+
s3Keys.map(setS3File(_))
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
def setRedirectObject(s3Key: String) {
|
|
1288
|
+
setS3File(s3Key, size = 0)
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
def setS3File(s3Key: String, md5: String = "", size: Int = 10) {
|
|
1292
|
+
s3ObjectListing.getObjectSummaries.add({
|
|
1293
|
+
val summary = new S3ObjectSummary
|
|
1294
|
+
summary.setETag(md5)
|
|
1295
|
+
summary.setKey(s3Key)
|
|
1296
|
+
summary.setSize(size)
|
|
1297
|
+
summary
|
|
1298
|
+
})
|
|
1299
|
+
}
|
|
1300
|
+
|
|
1301
|
+
def setS3Files(s3Files: S3File*) {
|
|
1302
|
+
s3Files.foreach { s3File =>
|
|
1303
|
+
setS3File(s3File.s3Key.key, s3File.md5)
|
|
1304
|
+
}
|
|
1305
|
+
}
|
|
1306
|
+
|
|
1307
|
+
def removeAllFilesFromS3() {
|
|
1308
|
+
setS3Files(Nil: _*) // This corresponds to the situation where the S3 bucket is empty
|
|
1309
|
+
}
|
|
1310
|
+
|
|
1311
|
+
def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
|
1312
|
+
doAnswer(temporaryFailure(classOf[PutObjectResult]))
|
|
1313
|
+
.when(amazonS3Client)
|
|
1314
|
+
.putObject(Matchers.anyObject())
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
def deleteFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
|
1318
|
+
doAnswer(temporaryFailure(classOf[DeleteObjectRequest]))
|
|
1319
|
+
.when(amazonS3Client)
|
|
1320
|
+
.deleteObject(Matchers.anyString(), Matchers.anyString())
|
|
1321
|
+
}
|
|
1322
|
+
|
|
1323
|
+
def objectListingFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
|
1324
|
+
doAnswer(temporaryFailure(classOf[ObjectListing]))
|
|
1325
|
+
.when(amazonS3Client)
|
|
1326
|
+
.listObjects(Matchers.any(classOf[ListObjectsRequest]))
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
def sentPutObjectRequests: Seq[PutObjectRequest] = {
|
|
1330
|
+
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
|
1331
|
+
verify(amazonS3Client, Mockito.atLeast(1)).putObject(req.capture())
|
|
1332
|
+
req.getAllValues
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
def sentPutObjectRequest = sentPutObjectRequests.ensuring(_.length == 1).head
|
|
1336
|
+
|
|
1337
|
+
def sentDeletes: Seq[S3Key] = {
|
|
1338
|
+
val deleteKey = ArgumentCaptor.forClass(classOf[S3Key])
|
|
1339
|
+
verify(amazonS3Client).deleteObject(Matchers.anyString(), deleteKey.capture())
|
|
1340
|
+
deleteKey.getAllValues
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
def sentDelete = sentDeletes.ensuring(_.length == 1).head
|
|
1344
|
+
|
|
1345
|
+
def noDeletesOccurred = {
|
|
1346
|
+
verify(amazonS3Client, never()).deleteObject(Matchers.anyString(), Matchers.anyString())
|
|
1347
|
+
true // Mockito is based on exceptions
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
def noUploadsOccurred = {
|
|
1351
|
+
verify(amazonS3Client, never()).putObject(Matchers.any(classOf[PutObjectRequest]))
|
|
1352
|
+
true // Mockito is based on exceptions
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
type S3Key = String
|
|
1356
|
+
}
|
|
1357
|
+
|
|
1358
|
+
trait MockAWSHelper {
|
|
1359
|
+
def temporaryFailure[T](clazz: Class[T])(implicit callCount: AtomicInteger, howManyFailures: Int) = new Answer[T] {
|
|
1360
|
+
def answer(invocation: InvocationOnMock) = {
|
|
1361
|
+
callCount.incrementAndGet()
|
|
1362
|
+
if (callCount.get() <= howManyFailures)
|
|
1363
|
+
throw new AmazonServiceException("AWS is temporarily down")
|
|
1364
|
+
else
|
|
1365
|
+
mock(clazz)
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
trait Directories extends BeforeAfter {
|
|
1371
|
+
def randomDir() = new File(FileUtils.getTempDirectory, "s3_website_dir" + Random.nextLong())
|
|
1372
|
+
implicit final val workingDirectory: File = randomDir()
|
|
1373
|
+
implicit def yamlConfig: S3_website_yml = S3_website_yml(new File(workingDirectory, "s3_website.yml"))
|
|
1374
|
+
val siteDirectory: File
|
|
1375
|
+
val configDirectory: File = workingDirectory // Represents the --config-dir=X option
|
|
1376
|
+
|
|
1377
|
+
def before {
|
|
1378
|
+
workingDirectory :: siteDirectory :: configDirectory :: Nil foreach forceMkdir
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
def after {
|
|
1382
|
+
(workingDirectory :: siteDirectory :: configDirectory :: Nil) foreach { dir =>
|
|
1383
|
+
if (dir.exists) forceDelete(dir)
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
|
|
1388
|
+
trait SiteLocationFromCliArg extends Directories {
|
|
1389
|
+
val siteDirectory = workingDirectory
|
|
1390
|
+
val siteDirFromCLIArg = true
|
|
1391
|
+
}
|
|
1392
|
+
|
|
1393
|
+
trait JekyllSite extends Directories {
|
|
1394
|
+
val siteDirectory = new File(workingDirectory, "_site")
|
|
1395
|
+
val siteDirFromCLIArg = false
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
trait NanocSite extends Directories {
|
|
1399
|
+
val siteDirectory = new File(workingDirectory, "public/output")
|
|
1400
|
+
val siteDirFromCLIArg = false
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
trait MiddlemanSite extends Directories {
|
|
1404
|
+
val siteDirectory = new File(workingDirectory, "build")
|
|
1405
|
+
val siteDirFromCLIArg = false
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
trait CustomSiteDirectory extends Directories {
|
|
1409
|
+
val siteDirectory = randomDir()
|
|
1410
|
+
val siteDirFromCLIArg = false
|
|
1411
|
+
}
|
|
1412
|
+
|
|
1413
|
+
trait EmptySite extends Directories {
|
|
1414
|
+
val siteDirFromCLIArg: Boolean
|
|
1415
|
+
type LocalFileWithContent = (String, String)
|
|
1416
|
+
|
|
1417
|
+
def setLocalFile(fileName: String) = setLocalFileWithContent((fileName, ""))
|
|
1418
|
+
def setLocalFiles(fileNames: String*) = fileNames foreach setLocalFile
|
|
1419
|
+
def setLocalFilesWithContent(fileNamesAndContent: LocalFileWithContent*) = fileNamesAndContent foreach setLocalFileWithContent
|
|
1420
|
+
def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) = {
|
|
1421
|
+
val file = new File(siteDirectory, fileNameAndContent._1)
|
|
1422
|
+
forceMkdir(file.getParentFile)
|
|
1423
|
+
file.createNewFile()
|
|
1424
|
+
write(file, fileNameAndContent._2)
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
def setLocalFileWithContent(fileName: String, contents: Array[Byte]) = {
|
|
1428
|
+
val file = new File(siteDirectory, fileName)
|
|
1429
|
+
forceMkdir(file.getParentFile)
|
|
1430
|
+
file.createNewFile()
|
|
1431
|
+
FileUtils.writeByteArrayToFile(file, contents)
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
var config = ""
|
|
1435
|
+
val baseConfig =
|
|
1436
|
+
"""
|
|
1437
|
+
|s3_id: foo
|
|
1438
|
+
|s3_secret: bar
|
|
1439
|
+
|s3_bucket: bucket
|
|
1440
|
+
""".stripMargin
|
|
1441
|
+
|
|
1442
|
+
implicit def configString: ConfigString =
|
|
1443
|
+
ConfigString(
|
|
1444
|
+
s"""
|
|
1445
|
+
|$baseConfig
|
|
1446
|
+
|$config
|
|
1447
|
+
""".stripMargin
|
|
1448
|
+
)
|
|
1449
|
+
|
|
1450
|
+
def pushOptions: PushOptions
|
|
1451
|
+
|
|
1452
|
+
implicit def cliArgs: CliArgs =
|
|
1453
|
+
new CliArgs {
|
|
1454
|
+
def verbose = true
|
|
1455
|
+
|
|
1456
|
+
def dryRun = pushOptions.dryRun
|
|
1457
|
+
|
|
1458
|
+
def site = if (siteDirFromCLIArg) siteDirectory.getAbsolutePath else null
|
|
1459
|
+
|
|
1460
|
+
def configDir = configDirectory.getAbsolutePath
|
|
1461
|
+
|
|
1462
|
+
def force = pushOptions.force
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
def push(logCapturer: Option[(String) => _] = None)
|
|
1467
|
+
(implicit
|
|
1468
|
+
emptyYamlConfig: S3_website_yml,
|
|
1469
|
+
configString: ConfigString,
|
|
1470
|
+
cliArgs: CliArgs,
|
|
1471
|
+
s3Settings: S3Setting,
|
|
1472
|
+
cloudFrontSettings: CloudFrontSetting,
|
|
1473
|
+
workingDirectory: File) = {
|
|
1474
|
+
write(emptyYamlConfig.file, configString.yaml) // Write the yaml config lazily, so that the tests can override the default yaml config
|
|
1475
|
+
implicit val logger = new Logger(verboseOutput = cliArgs.verbose, logCapturer)
|
|
1476
|
+
Push.push
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
case class ConfigString(yaml: String)
|
|
1480
|
+
}
|