s3_website_monadic 0.0.27 → 0.0.28
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -0
- data/bin/s3_website_monadic +13 -1
- data/lib/s3_website/version.rb +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +10 -10
- data/src/main/scala/s3/website/Push.scala +31 -21
- data/src/main/scala/s3/website/S3.scala +16 -15
- data/src/main/scala/s3/website/Utils.scala +30 -5
- data/src/main/scala/s3/website/model/push.scala +0 -3
- data/src/main/scala/s3/website/package.scala +37 -3
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +106 -65
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7900f28823f8541d10361f8b99020be7a403bb70
|
4
|
+
data.tar.gz: 2e0dad3c0abf21757fce241e11a7145f58d44cd4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 38a3e67004aeeb887118d7f8bfa7f5fef076bbc1c8d72ce6d6aaf409f51ed279c6602b51240ab7532bd45bf961189d2a75c1f25949e28d782b151d7b1cea8903
|
7
|
+
data.tar.gz: a1e09461dec20cec96591c7b67fe6ff4299a474e30b339972cedf49408e7629028e11ae0cb9ab5e7e67a12fdbd04f03af3f2e563ffa9a1a2ac739abd99bfa7e5
|
data/README.md
CHANGED
@@ -335,6 +335,16 @@ If you experience the "too many open files" error, either increase the amount of
|
|
335
335
|
maximum open files (on Unix-like systems, see `man ulimit`) or decrease the
|
336
336
|
`concurrency_level` setting.
|
337
337
|
|
338
|
+
## Simulating deployments
|
339
|
+
|
340
|
+
You can simulate the `s3_website_monadic push` operation by adding the
|
341
|
+
`--dry-run` switch. The dry run mode will not apply any modifications on your S3
|
342
|
+
bucket or CloudFront distribution. It will merely print out what the `push`
|
343
|
+
operation would actually do if run without the dry switch.
|
344
|
+
|
345
|
+
You can use the dry run mode if you are unsure what kind of effects the `push`
|
346
|
+
operation would cause to your live website.
|
347
|
+
|
338
348
|
## Example configurations
|
339
349
|
|
340
350
|
See
|
data/bin/s3_website_monadic
CHANGED
@@ -59,6 +59,12 @@ class Cli < Thor
|
|
59
59
|
:default => false,
|
60
60
|
:desc => "Print verbose output"
|
61
61
|
)
|
62
|
+
option(
|
63
|
+
:dry_run,
|
64
|
+
:type => :boolean,
|
65
|
+
:default => false,
|
66
|
+
:desc => "Run the operation without actually making the modifications. When this switch is on, changes will not be applied on the S3 website. They will be only printed to the console."
|
67
|
+
)
|
62
68
|
desc 'push', 'Push local files with the S3 website'
|
63
69
|
long_desc <<-LONGDESC
|
64
70
|
`s3_website push` will upload new and changes files to S3. It will
|
@@ -82,7 +88,13 @@ end
|
|
82
88
|
def run_s3_website_jar(jar_file, call_dir, logger)
|
83
89
|
site_path = File.expand_path(S3Website::Paths.infer_site_path options[:site], call_dir)
|
84
90
|
config_dir = File.expand_path options[:config_dir]
|
85
|
-
|
91
|
+
|
92
|
+
args = [
|
93
|
+
"--site=#{site_path}",
|
94
|
+
"--config-dir=#{config_dir}",
|
95
|
+
"#{'--verbose' if options[:verbose]}",
|
96
|
+
"#{'--dry-run' if options[:dry_run]}"
|
97
|
+
].join ' '
|
86
98
|
logger.debug_msg "Using #{jar_file}"
|
87
99
|
if system("java -cp #{jar_file} s3.website.Push #{args}")
|
88
100
|
exit 0
|
data/lib/s3_website/version.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
package s3.website
|
2
2
|
|
3
|
-
import s3.website.model.{Update,
|
3
|
+
import s3.website.model.{Update, Config}
|
4
4
|
import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
|
5
|
-
import s3.website.CloudFront.{
|
5
|
+
import s3.website.CloudFront.{CloudFrontSetting, SuccessfulInvalidation, FailedInvalidation}
|
6
6
|
import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
|
7
7
|
import scala.collection.JavaConversions._
|
8
8
|
import scala.concurrent.duration._
|
@@ -12,15 +12,15 @@ import java.net.URI
|
|
12
12
|
import Utils._
|
13
13
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
14
14
|
|
15
|
-
class CloudFront(implicit cloudFrontSettings:
|
15
|
+
class CloudFront(implicit cloudFrontSettings: CloudFrontSetting, config: Config, logger: Logger, pushMode: PushMode) {
|
16
16
|
val cloudFront = cloudFrontSettings.cfClient(config)
|
17
17
|
|
18
18
|
def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
|
19
19
|
(implicit ec: ExecutionContextExecutor): InvalidationResult =
|
20
20
|
Future {
|
21
|
-
|
22
|
-
cloudFront.createInvalidation(invalidationReq)
|
21
|
+
if (!pushMode.dryRun) cloudFront createInvalidation new CreateInvalidationRequest(distributionId, invalidationBatch)
|
23
22
|
val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
|
23
|
+
logger.debug(invalidationBatch.getPaths.getItems.map(item => s"${Invalidated.renderVerb} $item") mkString "\n")
|
24
24
|
logger.info(result)
|
25
25
|
Right(result)
|
26
26
|
} recoverWith (tooManyInvalidationsRetry(invalidationBatch, distributionId, attempt) orElse retry(attempt)(
|
@@ -44,7 +44,7 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config
|
|
44
44
|
val basicInfo = s"The maximum amount of CloudFront invalidations has exceeded. Trying again in $sleepDuration, please wait."
|
45
45
|
val extendedInfo =
|
46
46
|
s"""|$basicInfo
|
47
|
-
|
|
47
|
+
|For more information, see http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits"""
|
48
48
|
.stripMargin
|
49
49
|
if (attempt == 1)
|
50
50
|
extendedInfo
|
@@ -59,8 +59,8 @@ object CloudFront {
|
|
59
59
|
|
60
60
|
type CloudFrontClientProvider = (Config) => AmazonCloudFront
|
61
61
|
|
62
|
-
case class SuccessfulInvalidation(invalidatedItemsCount: Int) extends SuccessReport {
|
63
|
-
def reportMessage = s"Invalidated ${invalidatedItemsCount ofType "item"} on CloudFront"
|
62
|
+
case class SuccessfulInvalidation(invalidatedItemsCount: Int)(implicit pushMode: PushMode) extends SuccessReport {
|
63
|
+
def reportMessage = s"${Invalidated.renderVerb} ${invalidatedItemsCount ofType "item"} on CloudFront"
|
64
64
|
}
|
65
65
|
|
66
66
|
case class FailedInvalidation(error: Throwable) extends FailureReport {
|
@@ -129,8 +129,8 @@ object CloudFront {
|
|
129
129
|
case _ => false
|
130
130
|
}
|
131
131
|
|
132
|
-
case class
|
132
|
+
case class CloudFrontSetting(
|
133
133
|
cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
|
134
134
|
retryTimeUnit: TimeUnit = MINUTES
|
135
|
-
) extends
|
135
|
+
) extends RetrySetting
|
136
136
|
}
|
@@ -20,21 +20,23 @@ import scala.collection.mutable.ArrayBuffer
|
|
20
20
|
import s3.website.CloudFront._
|
21
21
|
import s3.website.S3.SuccessfulDelete
|
22
22
|
import s3.website.CloudFront.SuccessfulInvalidation
|
23
|
-
import s3.website.S3.
|
24
|
-
import s3.website.CloudFront.
|
23
|
+
import s3.website.S3.S3Setting
|
24
|
+
import s3.website.CloudFront.CloudFrontSetting
|
25
25
|
import s3.website.S3.SuccessfulUpload
|
26
26
|
import s3.website.CloudFront.FailedInvalidation
|
27
|
+
import scala.Int
|
27
28
|
|
28
29
|
object Push {
|
29
30
|
|
30
31
|
def pushSite(
|
31
32
|
implicit site: Site,
|
32
33
|
executor: ExecutionContextExecutor,
|
33
|
-
s3Settings:
|
34
|
-
cloudFrontSettings:
|
35
|
-
logger: Logger
|
34
|
+
s3Settings: S3Setting,
|
35
|
+
cloudFrontSettings: CloudFrontSetting,
|
36
|
+
logger: Logger,
|
37
|
+
pushMode: PushMode
|
36
38
|
): ExitCode = {
|
37
|
-
logger.info(s"
|
39
|
+
logger.info(s"${Deploy.renderVerb} ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
38
40
|
val utils = new Utils
|
39
41
|
|
40
42
|
val redirects = Redirect.resolveRedirects
|
@@ -63,7 +65,8 @@ object Push {
|
|
63
65
|
|
64
66
|
def invalidateCloudFrontItems
|
65
67
|
(errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
|
66
|
-
(implicit config: Config, cloudFrontSettings:
|
68
|
+
(implicit config: Config, cloudFrontSettings: CloudFrontSetting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
|
69
|
+
Option[InvalidationSucceeded] = {
|
67
70
|
config.cloudfront_distribution_id.map {
|
68
71
|
distributionId =>
|
69
72
|
val pushSuccessReports = errorsOrFinishedPushOps.fold(
|
@@ -101,7 +104,7 @@ object Push {
|
|
101
104
|
type InvalidationSucceeded = Boolean
|
102
105
|
|
103
106
|
def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])
|
104
|
-
(implicit config: Config, logger: Logger): ExitCode = {
|
107
|
+
(implicit config: Config, logger: Logger, pushMode: PushMode): ExitCode = {
|
105
108
|
errorsOrFinishedUploads.right.foreach { finishedUploads =>
|
106
109
|
val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
|
107
110
|
logger.info(s"Summary: $pushCounts")
|
@@ -120,10 +123,13 @@ object Push {
|
|
120
123
|
if (allInvalidationsSucceeded) 0 else 1
|
121
124
|
)
|
122
125
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
126
|
+
exitCode match {
|
127
|
+
case 0 if !pushMode.dryRun =>
|
128
|
+
logger.info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
129
|
+
case 1 =>
|
130
|
+
logger.fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
131
|
+
case _ =>
|
132
|
+
}
|
127
133
|
exitCode
|
128
134
|
}
|
129
135
|
|
@@ -149,18 +155,18 @@ object Push {
|
|
149
155
|
)
|
150
156
|
}
|
151
157
|
|
152
|
-
def pushCountsToString(pushCounts: PushCounts): String =
|
158
|
+
def pushCountsToString(pushCounts: PushCounts)(implicit pushMode: PushMode): String =
|
153
159
|
pushCounts match {
|
154
160
|
case PushCounts(updates, newFiles, failures, redirects, deletes)
|
155
161
|
if updates == 0 && newFiles == 0 && failures == 0 && redirects == 0 && deletes == 0 =>
|
156
|
-
|
162
|
+
PushNothing.renderVerb
|
157
163
|
case PushCounts(updates, newFiles, failures, redirects, deletes) =>
|
158
164
|
val reportClauses: scala.collection.mutable.ArrayBuffer[String] = ArrayBuffer()
|
159
|
-
if (updates > 0) reportClauses += s"Updated ${updates ofType "file"}."
|
160
|
-
if (newFiles > 0) reportClauses += s"Created ${newFiles ofType "file"}."
|
165
|
+
if (updates > 0) reportClauses += s"${Updated.renderVerb} ${updates ofType "file"}."
|
166
|
+
if (newFiles > 0) reportClauses += s"${Created.renderVerb} ${newFiles ofType "file"}."
|
161
167
|
if (failures > 0) reportClauses += s"${failures ofType "operation"} failed." // This includes both failed uploads and deletes.
|
162
|
-
if (redirects > 0) reportClauses += s"Applied ${redirects ofType "redirect"}."
|
163
|
-
if (deletes > 0) reportClauses += s"Deleted ${deletes ofType "file"}."
|
168
|
+
if (redirects > 0) reportClauses += s"${Applied.renderVerb} ${redirects ofType "redirect"}."
|
169
|
+
if (deletes > 0) reportClauses += s"${Deleted.renderVerb} ${deletes ofType "file"}."
|
164
170
|
reportClauses.mkString(" ")
|
165
171
|
}
|
166
172
|
|
@@ -182,20 +188,24 @@ object Push {
|
|
182
188
|
@Option def site: String
|
183
189
|
@Option(longName = Array("config-dir")) def configDir: String
|
184
190
|
@Option def verbose: Boolean
|
191
|
+
@Option(longName = Array("dry-run")) def dryRun: Boolean
|
185
192
|
}
|
186
193
|
|
187
194
|
def main(args: Array[String]) {
|
188
195
|
val cliArgs = CliFactory.parseArguments(classOf[CliArgs], args:_*)
|
189
|
-
implicit val s3Settings =
|
190
|
-
implicit val cloudFrontSettings =
|
196
|
+
implicit val s3Settings = S3Setting()
|
197
|
+
implicit val cloudFrontSettings = CloudFrontSetting()
|
191
198
|
implicit val logger: Logger = new Logger(cliArgs.verbose)
|
199
|
+
implicit val pushMode = new PushMode {
|
200
|
+
def dryRun = cliArgs.dryRun
|
201
|
+
}
|
192
202
|
val errorOrPushStatus = push(siteInDirectory = cliArgs.site, withConfigDirectory = cliArgs.configDir)
|
193
203
|
errorOrPushStatus.left foreach (err => logger.fail(s"Could not load the site: ${err.reportMessage}"))
|
194
204
|
System exit errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus)
|
195
205
|
}
|
196
206
|
|
197
207
|
def push(siteInDirectory: String, withConfigDirectory: String)
|
198
|
-
(implicit s3Settings:
|
208
|
+
(implicit s3Settings: S3Setting, cloudFrontSettings: CloudFrontSetting, logger: Logger, pushMode: PushMode) =
|
199
209
|
loadSite(withConfigDirectory + "/s3_website.yml", siteInDirectory)
|
200
210
|
.right
|
201
211
|
.map {
|
@@ -14,15 +14,15 @@ import s3.website.S3.SuccessfulDelete
|
|
14
14
|
import s3.website.S3.FailedUpload
|
15
15
|
import scala.Some
|
16
16
|
import s3.website.S3.FailedDelete
|
17
|
-
import s3.website.S3.
|
17
|
+
import s3.website.S3.S3Setting
|
18
18
|
|
19
|
-
class S3(implicit s3Settings:
|
19
|
+
class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor) {
|
20
20
|
|
21
21
|
def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)
|
22
22
|
(implicit config: Config, logger: Logger): Future[Either[FailedUpload, SuccessfulUpload]] =
|
23
23
|
Future {
|
24
24
|
val putObjectRequest = toPutObjectRequest(upload)
|
25
|
-
s3Settings.s3Client(config) putObject putObjectRequest
|
25
|
+
if (!pushMode.dryRun) s3Settings.s3Client(config) putObject putObjectRequest
|
26
26
|
val report = SuccessfulUpload(upload, putObjectRequest)
|
27
27
|
logger.info(report)
|
28
28
|
Right(report)
|
@@ -34,7 +34,7 @@ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
|
|
34
34
|
def delete(s3Key: String, a: Attempt = 1)
|
35
35
|
(implicit config: Config, logger: Logger): Future[Either[FailedDelete, SuccessfulDelete]] =
|
36
36
|
Future {
|
37
|
-
s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
37
|
+
if (!pushMode.dryRun) s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
38
38
|
val report = SuccessfulDelete(s3Key)
|
39
39
|
logger.info(report)
|
40
40
|
Right(report)
|
@@ -84,7 +84,7 @@ object S3 {
|
|
84
84
|
|
85
85
|
def resolveS3FilesAndUpdates(localFiles: Seq[LocalFile])
|
86
86
|
(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil, attempt: Attempt = 1, onFlightUpdateFutures: UpdateFutures = Nil)
|
87
|
-
(implicit config: Config, s3Settings:
|
87
|
+
(implicit config: Config, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
|
88
88
|
ErrorOrS3FilesAndUpdates = Future {
|
89
89
|
logger.debug(nextMarker.fold
|
90
90
|
("Querying S3 files")
|
@@ -97,12 +97,12 @@ object S3 {
|
|
97
97
|
req
|
98
98
|
})
|
99
99
|
val summaryIndex = objects.getObjectSummaries.map { summary => (summary.getETag, summary.getKey) }.toSet // Index to avoid O(n^2) lookups
|
100
|
-
def
|
100
|
+
def shouldUpdate(lf: LocalFile) =
|
101
101
|
summaryIndex.exists((md5AndS3Key) =>
|
102
102
|
md5AndS3Key._1 != lf.md5 && md5AndS3Key._2 == lf.s3Key
|
103
103
|
)
|
104
104
|
val updateFutures: UpdateFutures = localFiles.collect {
|
105
|
-
case lf: LocalFile if
|
105
|
+
case lf: LocalFile if shouldUpdate(lf) =>
|
106
106
|
val errorOrUpdate = LocalFile
|
107
107
|
.toUpload(lf)
|
108
108
|
.right
|
@@ -138,7 +138,8 @@ object S3 {
|
|
138
138
|
def s3Key: String
|
139
139
|
}
|
140
140
|
|
141
|
-
case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest)
|
141
|
+
case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest)
|
142
|
+
(implicit pushMode: PushMode) extends PushSuccessReport {
|
142
143
|
val metadata = putObjectRequest.getMetadata
|
143
144
|
def metadataReport =
|
144
145
|
(metadata.getCacheControl :: metadata.getContentType :: metadata.getContentEncoding :: putObjectRequest.getStorageClass :: Nil)
|
@@ -147,16 +148,16 @@ object S3 {
|
|
147
148
|
|
148
149
|
def reportMessage =
|
149
150
|
upload.uploadType match {
|
150
|
-
case NewFile => s"Created ${upload.s3Key} ($metadataReport)"
|
151
|
-
case Update => s"Updated ${upload.s3Key} ($metadataReport)"
|
152
|
-
case Redirect => s"Redirected ${upload.essence.left.get.key} to ${upload.essence.left.get.redirectTarget}"
|
151
|
+
case NewFile => s"${Created.renderVerb} ${upload.s3Key} ($metadataReport)"
|
152
|
+
case Update => s"${Updated.renderVerb} ${upload.s3Key} ($metadataReport)"
|
153
|
+
case Redirect => s"${Redirected.renderVerb} ${upload.essence.left.get.key} to ${upload.essence.left.get.redirectTarget}"
|
153
154
|
}
|
154
155
|
|
155
156
|
def s3Key = upload.s3Key
|
156
157
|
}
|
157
158
|
|
158
|
-
case class SuccessfulDelete(s3Key: String) extends PushSuccessReport {
|
159
|
-
def reportMessage = s"Deleted $s3Key"
|
159
|
+
case class SuccessfulDelete(s3Key: String)(implicit pushMode: PushMode) extends PushSuccessReport {
|
160
|
+
def reportMessage = s"${Deleted.renderVerb} $s3Key"
|
160
161
|
}
|
161
162
|
|
162
163
|
case class FailedUpload(s3Key: String, error: Throwable) extends PushFailureReport {
|
@@ -169,8 +170,8 @@ object S3 {
|
|
169
170
|
|
170
171
|
type S3ClientProvider = (Config) => AmazonS3
|
171
172
|
|
172
|
-
case class
|
173
|
+
case class S3Setting(
|
173
174
|
s3Client: S3ClientProvider = S3.awsS3Client,
|
174
175
|
retryTimeUnit: TimeUnit = SECONDS
|
175
|
-
) extends
|
176
|
+
) extends RetrySetting
|
176
177
|
}
|
@@ -18,14 +18,39 @@ object Utils {
|
|
18
18
|
|
19
19
|
class Logger(verboseOutput: Boolean, logMessage: (String) => Unit = println) {
|
20
20
|
import Rainbow._
|
21
|
-
def debug(msg: String) = if (verboseOutput)
|
22
|
-
def info(msg: String) =
|
23
|
-
def fail(msg: String) =
|
21
|
+
def debug(msg: String) = if (verboseOutput) log(Debug, msg)
|
22
|
+
def info(msg: String) = log(Info, msg)
|
23
|
+
def fail(msg: String) = log(Failure, msg)
|
24
24
|
|
25
|
-
def info(report: SuccessReport) =
|
25
|
+
def info(report: SuccessReport) = log(Success, report.reportMessage)
|
26
26
|
def info(report: FailureReport) = fail(report.reportMessage)
|
27
27
|
|
28
|
-
def pending(msg: String) =
|
28
|
+
def pending(msg: String) = log(Wait, msg)
|
29
|
+
|
30
|
+
private def log(logType: LogType, msgRaw: String) {
|
31
|
+
val msg = msgRaw.replaceAll("\\n", "\n ") // Indent new lines, so that they arrange nicely with other log lines
|
32
|
+
logMessage(s"[$logType] $msg")
|
33
|
+
}
|
34
|
+
|
35
|
+
sealed trait LogType {
|
36
|
+
val prefix: String
|
37
|
+
override def toString = prefix
|
38
|
+
}
|
39
|
+
case object Debug extends LogType {
|
40
|
+
val prefix = "debg".cyan
|
41
|
+
}
|
42
|
+
case object Info extends LogType {
|
43
|
+
val prefix = "info".blue
|
44
|
+
}
|
45
|
+
case object Success extends LogType {
|
46
|
+
val prefix = "succ".green
|
47
|
+
}
|
48
|
+
case object Failure extends LogType {
|
49
|
+
val prefix = "fail".red
|
50
|
+
}
|
51
|
+
case object Wait extends LogType {
|
52
|
+
val prefix = "wait".yellow
|
53
|
+
}
|
29
54
|
}
|
30
55
|
|
31
56
|
/**
|
@@ -17,9 +17,43 @@ package object website {
|
|
17
17
|
|
18
18
|
trait ErrorReport extends Report
|
19
19
|
|
20
|
-
trait
|
20
|
+
trait RetrySetting {
|
21
21
|
def retryTimeUnit: TimeUnit
|
22
22
|
}
|
23
|
+
|
24
|
+
trait PushMode {
|
25
|
+
def dryRun: Boolean
|
26
|
+
}
|
27
|
+
|
28
|
+
trait PushAction {
|
29
|
+
def actionName = getClass.getSimpleName.replace("$", "") // case object class names contain the '$' char
|
30
|
+
|
31
|
+
def renderVerb(implicit pushMode: PushMode): String =
|
32
|
+
if (pushMode.dryRun)
|
33
|
+
s"Would have ${actionName.toLowerCase}"
|
34
|
+
else
|
35
|
+
s"$actionName"
|
36
|
+
}
|
37
|
+
case object Created extends PushAction
|
38
|
+
case object Updated extends PushAction
|
39
|
+
case object Redirected extends PushAction
|
40
|
+
case object Deleted extends PushAction
|
41
|
+
case object Invalidated extends PushAction
|
42
|
+
case object Applied extends PushAction
|
43
|
+
case object PushNothing extends PushAction {
|
44
|
+
override def renderVerb(implicit pushMode: PushMode) =
|
45
|
+
if (pushMode.dryRun)
|
46
|
+
s"Would have pushed nothing"
|
47
|
+
else
|
48
|
+
s"There was nothing to push"
|
49
|
+
}
|
50
|
+
case object Deploy extends PushAction {
|
51
|
+
override def renderVerb(implicit pushMode: PushMode) =
|
52
|
+
if (pushMode.dryRun)
|
53
|
+
s"Simulating the deployment of"
|
54
|
+
else
|
55
|
+
s"Deploying"
|
56
|
+
}
|
23
57
|
|
24
58
|
type PushErrorOrSuccess = Either[PushFailureReport, PushSuccessReport]
|
25
59
|
|
@@ -27,7 +61,7 @@ package object website {
|
|
27
61
|
|
28
62
|
def retry[L <: Report, R](attempt: Attempt)
|
29
63
|
(createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
|
30
|
-
(implicit
|
64
|
+
(implicit retrySetting: RetrySetting, ec: ExecutionContextExecutor, logger: Logger):
|
31
65
|
PartialFunction[Throwable, Future[Either[L, R]]] = {
|
32
66
|
case error: Throwable if attempt == 6 || isIrrecoverable(error) =>
|
33
67
|
val failureReport = createFailureReport(error)
|
@@ -35,7 +69,7 @@ package object website {
|
|
35
69
|
Future(Left(failureReport))
|
36
70
|
case error: Throwable =>
|
37
71
|
val failureReport = createFailureReport(error)
|
38
|
-
val sleepDuration = Duration(fibs.drop(attempt + 1).head,
|
72
|
+
val sleepDuration = Duration(fibs.drop(attempt + 1).head, retrySetting.retryTimeUnit)
|
39
73
|
logger.pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
40
74
|
Thread.sleep(sleepDuration.toMillis)
|
41
75
|
retryAction(attempt + 1)
|
@@ -11,14 +11,12 @@ import org.mockito.Mockito._
|
|
11
11
|
import com.amazonaws.services.s3.AmazonS3
|
12
12
|
import com.amazonaws.services.s3.model._
|
13
13
|
import scala.concurrent.ExecutionContext.Implicits.global
|
14
|
-
import scala.concurrent.Await
|
15
14
|
import scala.concurrent.duration._
|
16
|
-
import s3.website.S3.
|
15
|
+
import s3.website.S3.S3Setting
|
17
16
|
import scala.collection.JavaConversions._
|
18
|
-
import s3.website.model.NewFile
|
19
17
|
import com.amazonaws.AmazonServiceException
|
20
18
|
import org.apache.commons.codec.digest.DigestUtils.md5Hex
|
21
|
-
import s3.website.CloudFront.
|
19
|
+
import s3.website.CloudFront.CloudFrontSetting
|
22
20
|
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
23
21
|
import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
|
24
22
|
import org.mockito.stubbing.Answer
|
@@ -30,7 +28,7 @@ import scala.collection.mutable
|
|
30
28
|
class S3WebsiteSpec extends Specification {
|
31
29
|
|
32
30
|
"gzip: true" should {
|
33
|
-
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS {
|
31
|
+
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
34
32
|
config = "gzip: true"
|
35
33
|
setLocalFileWithContent(("styles.css", "<h1>hi again</h1>"))
|
36
34
|
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
@@ -38,7 +36,7 @@ class S3WebsiteSpec extends Specification {
|
|
38
36
|
sentPutObjectRequest.getKey must equalTo("styles.css")
|
39
37
|
}
|
40
38
|
|
41
|
-
"not update a gzipped S3 object if the contents has not changed" in new EmptySite with VerboseLogger with MockAWS {
|
39
|
+
"not update a gzipped S3 object if the contents has not changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
42
40
|
config = "gzip: true"
|
43
41
|
setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
|
44
42
|
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
@@ -51,7 +49,7 @@ class S3WebsiteSpec extends Specification {
|
|
51
49
|
gzip:
|
52
50
|
- .xml
|
53
51
|
""" should {
|
54
|
-
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS {
|
52
|
+
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
55
53
|
config = """
|
56
54
|
|gzip:
|
57
55
|
| - .xml
|
@@ -64,40 +62,40 @@ class S3WebsiteSpec extends Specification {
|
|
64
62
|
}
|
65
63
|
|
66
64
|
"push" should {
|
67
|
-
"not upload a file if it has not changed" in new EmptySite with VerboseLogger with MockAWS {
|
65
|
+
"not upload a file if it has not changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
68
66
|
setLocalFileWithContent(("index.html", "<div>hello</div>"))
|
69
67
|
setS3Files(S3File("index.html", md5Hex("<div>hello</div>")))
|
70
68
|
Push.pushSite
|
71
69
|
noUploadsOccurred must beTrue
|
72
70
|
}
|
73
71
|
|
74
|
-
"update a file if it has changed" in new EmptySite with VerboseLogger with MockAWS {
|
72
|
+
"update a file if it has changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
75
73
|
setLocalFileWithContent(("index.html", "<h1>old text</h1>"))
|
76
74
|
setS3Files(S3File("index.html", md5Hex("<h1>new text</h1>")))
|
77
75
|
Push.pushSite
|
78
76
|
sentPutObjectRequest.getKey must equalTo("index.html")
|
79
77
|
}
|
80
78
|
|
81
|
-
"create a file if does not exist on S3" in new EmptySite with VerboseLogger with MockAWS {
|
79
|
+
"create a file if does not exist on S3" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
82
80
|
setLocalFile("index.html")
|
83
81
|
Push.pushSite
|
84
82
|
sentPutObjectRequest.getKey must equalTo("index.html")
|
85
83
|
}
|
86
84
|
|
87
|
-
"delete files that are on S3 but not on local file system" in new EmptySite with VerboseLogger with MockAWS {
|
85
|
+
"delete files that are on S3 but not on local file system" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
88
86
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
89
87
|
Push.pushSite
|
90
88
|
sentDelete must equalTo("old.html")
|
91
89
|
}
|
92
90
|
|
93
|
-
"try again if the upload fails" in new EmptySite with VerboseLogger with MockAWS {
|
91
|
+
"try again if the upload fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
94
92
|
setLocalFile("index.html")
|
95
93
|
uploadFailsAndThenSucceeds(howManyFailures = 5)
|
96
94
|
Push.pushSite
|
97
95
|
verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
98
96
|
}
|
99
97
|
|
100
|
-
"not try again if the upload fails on because of invalid credentials" in new EmptySite with VerboseLogger with MockAWS {
|
98
|
+
"not try again if the upload fails on because of invalid credentials" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
101
99
|
setLocalFile("index.html")
|
102
100
|
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
|
103
101
|
val e = new AmazonServiceException("your credentials are incorrect")
|
@@ -108,14 +106,14 @@ class S3WebsiteSpec extends Specification {
|
|
108
106
|
verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
109
107
|
}
|
110
108
|
|
111
|
-
"try again if the delete fails" in new EmptySite with VerboseLogger with MockAWS {
|
109
|
+
"try again if the delete fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
112
110
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
113
111
|
deleteFailsAndThenSucceeds(howManyFailures = 5)
|
114
112
|
Push.pushSite
|
115
113
|
verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
|
116
114
|
}
|
117
115
|
|
118
|
-
"try again if the object listing fails" in new EmptySite with VerboseLogger with MockAWS {
|
116
|
+
"try again if the object listing fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
119
117
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
120
118
|
objectListingFailsAndThenSucceeds(howManyFailures = 5)
|
121
119
|
Push.pushSite
|
@@ -124,7 +122,7 @@ class S3WebsiteSpec extends Specification {
|
|
124
122
|
}
|
125
123
|
|
126
124
|
"push with CloudFront" should {
|
127
|
-
"invalidate the updated CloudFront items" in new EmptySite with VerboseLogger with MockAWS {
|
125
|
+
"invalidate the updated CloudFront items" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
128
126
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
129
127
|
setLocalFiles("css/test.css", "articles/index.html")
|
130
128
|
setOutdatedS3Keys("css/test.css", "articles/index.html")
|
@@ -132,14 +130,14 @@ class S3WebsiteSpec extends Specification {
|
|
132
130
|
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/css/test.css" :: "/articles/index.html" :: Nil).sorted)
|
133
131
|
}
|
134
132
|
|
135
|
-
"not send CloudFront invalidation requests on new objects" in new EmptySite with VerboseLogger with MockAWS {
|
133
|
+
"not send CloudFront invalidation requests on new objects" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
136
134
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
137
135
|
setLocalFile("newfile.js")
|
138
136
|
Push.pushSite
|
139
137
|
noInvalidationsOccurred must beTrue
|
140
138
|
}
|
141
139
|
|
142
|
-
"not send CloudFront invalidation requests on redirect objects" in new EmptySite with VerboseLogger with MockAWS {
|
140
|
+
"not send CloudFront invalidation requests on redirect objects" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
143
141
|
config = """
|
144
142
|
|cloudfront_distribution_id: EGM1J2JJX9Z
|
145
143
|
|redirects:
|
@@ -149,7 +147,7 @@ class S3WebsiteSpec extends Specification {
|
|
149
147
|
noInvalidationsOccurred must beTrue
|
150
148
|
}
|
151
149
|
|
152
|
-
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new EmptySite with VerboseLogger with MockAWS {
|
150
|
+
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
153
151
|
setTooManyInvalidationsInProgress(4)
|
154
152
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
155
153
|
setLocalFile("test.css")
|
@@ -158,7 +156,7 @@ class S3WebsiteSpec extends Specification {
|
|
158
156
|
sentInvalidationRequests.length must equalTo(4)
|
159
157
|
}
|
160
158
|
|
161
|
-
"retry if CloudFront is temporarily unreachable" in new EmptySite with VerboseLogger with MockAWS {
|
159
|
+
"retry if CloudFront is temporarily unreachable" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
162
160
|
invalidationsFailAndThenSucceed(5)
|
163
161
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
164
162
|
setLocalFile("test.css")
|
@@ -167,7 +165,7 @@ class S3WebsiteSpec extends Specification {
|
|
167
165
|
sentInvalidationRequests.length must equalTo(6)
|
168
166
|
}
|
169
167
|
|
170
|
-
"encode unsafe characters in the keys" in new EmptySite with VerboseLogger with MockAWS {
|
168
|
+
"encode unsafe characters in the keys" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
171
169
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
172
170
|
setLocalFile("articles/arnold's file.html")
|
173
171
|
setOutdatedS3Keys("articles/arnold's file.html")
|
@@ -175,11 +173,7 @@ class S3WebsiteSpec extends Specification {
|
|
175
173
|
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/arnold's%20file.html" :: Nil).sorted)
|
176
174
|
}
|
177
175
|
|
178
|
-
|
179
|
-
* Because CloudFront supports Default Root Objects (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DefaultRootObject.html),
|
180
|
-
* we have to guess
|
181
|
-
*/
|
182
|
-
"invalidate the root object '/' if a top-level object is updated or deleted" in new EmptySite with VerboseLogger with MockAWS {
|
176
|
+
"invalidate the root object '/' if a top-level object is updated or deleted" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
183
177
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
184
178
|
setLocalFile("maybe-index.html")
|
185
179
|
setOutdatedS3Keys("maybe-index.html")
|
@@ -189,7 +183,7 @@ class S3WebsiteSpec extends Specification {
|
|
189
183
|
}
|
190
184
|
|
191
185
|
"cloudfront_invalidate_root: true" should {
|
192
|
-
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new EmptySite with VerboseLogger with MockAWS {
|
186
|
+
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
193
187
|
config = """
|
194
188
|
|cloudfront_distribution_id: EGM1J2JJX9Z
|
195
189
|
|cloudfront_invalidate_root: true
|
@@ -202,7 +196,7 @@ class S3WebsiteSpec extends Specification {
|
|
202
196
|
}
|
203
197
|
|
204
198
|
"a site with over 1000 items" should {
|
205
|
-
"split the CloudFront invalidation requests into batches of 1000 items" in new EmptySite with VerboseLogger with MockAWS {
|
199
|
+
"split the CloudFront invalidation requests into batches of 1000 items" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
206
200
|
val files = (1 to 1002).map { i => s"lots-of-files/file-$i"}
|
207
201
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
208
202
|
setLocalFiles(files:_*)
|
@@ -215,18 +209,18 @@ class S3WebsiteSpec extends Specification {
|
|
215
209
|
}
|
216
210
|
|
217
211
|
"push exit status" should {
|
218
|
-
"be 0 all uploads succeed" in new EmptySite with VerboseLogger with MockAWS {
|
212
|
+
"be 0 all uploads succeed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
219
213
|
setLocalFiles("file.txt")
|
220
214
|
Push.pushSite must equalTo(0)
|
221
215
|
}
|
222
216
|
|
223
|
-
"be 1 if any of the uploads fails" in new EmptySite with VerboseLogger with MockAWS {
|
217
|
+
"be 1 if any of the uploads fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
224
218
|
setLocalFiles("file.txt")
|
225
219
|
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
226
220
|
Push.pushSite must equalTo(1)
|
227
221
|
}
|
228
222
|
|
229
|
-
"be 1 if any of the redirects fails" in new EmptySite with VerboseLogger with MockAWS {
|
223
|
+
"be 1 if any of the redirects fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
230
224
|
config = """
|
231
225
|
|redirects:
|
232
226
|
| index.php: /index.html
|
@@ -235,14 +229,14 @@ class S3WebsiteSpec extends Specification {
|
|
235
229
|
Push.pushSite must equalTo(1)
|
236
230
|
}
|
237
231
|
|
238
|
-
"be 0 if CloudFront invalidations and uploads succeed"in new EmptySite with VerboseLogger with MockAWS {
|
232
|
+
"be 0 if CloudFront invalidations and uploads succeed"in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
239
233
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
240
234
|
setLocalFile("test.css")
|
241
235
|
setOutdatedS3Keys("test.css")
|
242
236
|
Push.pushSite must equalTo(0)
|
243
237
|
}
|
244
238
|
|
245
|
-
"be 1 if CloudFront is unreachable or broken"in new EmptySite with VerboseLogger with MockAWS {
|
239
|
+
"be 1 if CloudFront is unreachable or broken"in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
246
240
|
setCloudFrontAsInternallyBroken()
|
247
241
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
248
242
|
setLocalFile("test.css")
|
@@ -250,19 +244,19 @@ class S3WebsiteSpec extends Specification {
|
|
250
244
|
Push.pushSite must equalTo(1)
|
251
245
|
}
|
252
246
|
|
253
|
-
"be 0 if upload retry succeeds" in new EmptySite with VerboseLogger with MockAWS {
|
247
|
+
"be 0 if upload retry succeeds" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
254
248
|
setLocalFile("index.html")
|
255
249
|
uploadFailsAndThenSucceeds(howManyFailures = 1)
|
256
250
|
Push.pushSite must equalTo(0)
|
257
251
|
}
|
258
252
|
|
259
|
-
"be 1 if delete retry fails" in new EmptySite with VerboseLogger with MockAWS {
|
253
|
+
"be 1 if delete retry fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
260
254
|
setLocalFile("index.html")
|
261
255
|
uploadFailsAndThenSucceeds(howManyFailures = 6)
|
262
256
|
Push.pushSite must equalTo(1)
|
263
257
|
}
|
264
258
|
|
265
|
-
"be 1 if an object listing fails" in new EmptySite with VerboseLogger with MockAWS {
|
259
|
+
"be 1 if an object listing fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
266
260
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
267
261
|
objectListingFailsAndThenSucceeds(howManyFailures = 6)
|
268
262
|
Push.pushSite must equalTo(1)
|
@@ -270,7 +264,7 @@ class S3WebsiteSpec extends Specification {
|
|
270
264
|
}
|
271
265
|
|
272
266
|
"s3_website.yml file" should {
|
273
|
-
"never be uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
267
|
+
"never be uploaded" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
274
268
|
setLocalFile("s3_website.yml")
|
275
269
|
Push.pushSite
|
276
270
|
noUploadsOccurred must beTrue
|
@@ -278,7 +272,7 @@ class S3WebsiteSpec extends Specification {
|
|
278
272
|
}
|
279
273
|
|
280
274
|
"exclude_from_upload: string" should {
|
281
|
-
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
275
|
+
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
282
276
|
config = "exclude_from_upload: .DS_.*?"
|
283
277
|
setLocalFile(".DS_Store")
|
284
278
|
Push.pushSite
|
@@ -291,7 +285,7 @@ class S3WebsiteSpec extends Specification {
|
|
291
285
|
- regex
|
292
286
|
- another_exclusion
|
293
287
|
""" should {
|
294
|
-
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
288
|
+
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
295
289
|
config = """
|
296
290
|
|exclude_from_upload:
|
297
291
|
| - .DS_.*?
|
@@ -304,7 +298,7 @@ class S3WebsiteSpec extends Specification {
|
|
304
298
|
}
|
305
299
|
|
306
300
|
"ignore_on_server: value" should {
|
307
|
-
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS {
|
301
|
+
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
308
302
|
config = "ignore_on_server: logs"
|
309
303
|
setS3Files(S3File("logs/log.txt", ""))
|
310
304
|
Push.pushSite
|
@@ -317,7 +311,7 @@ class S3WebsiteSpec extends Specification {
|
|
317
311
|
- regex
|
318
312
|
- another_ignore
|
319
313
|
""" should {
|
320
|
-
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS {
|
314
|
+
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
321
315
|
config = """
|
322
316
|
|ignore_on_server:
|
323
317
|
| - .*txt
|
@@ -329,14 +323,14 @@ class S3WebsiteSpec extends Specification {
|
|
329
323
|
}
|
330
324
|
|
331
325
|
"max-age in config" can {
|
332
|
-
"be applied to all files" in new EmptySite with VerboseLogger with MockAWS {
|
326
|
+
"be applied to all files" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
333
327
|
config = "max_age: 60"
|
334
328
|
setLocalFile("index.html")
|
335
329
|
Push.pushSite
|
336
330
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
337
331
|
}
|
338
332
|
|
339
|
-
"be applied to files that match the glob" in new EmptySite with VerboseLogger with MockAWS {
|
333
|
+
"be applied to files that match the glob" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
340
334
|
config = """
|
341
335
|
|max_age:
|
342
336
|
| "*.html": 90
|
@@ -346,7 +340,7 @@ class S3WebsiteSpec extends Specification {
|
|
346
340
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
347
341
|
}
|
348
342
|
|
349
|
-
"be applied to directories that match the glob" in new EmptySite with VerboseLogger with MockAWS {
|
343
|
+
"be applied to directories that match the glob" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
350
344
|
config = """
|
351
345
|
|max_age:
|
352
346
|
| "assets/**/*.js": 90
|
@@ -356,7 +350,7 @@ class S3WebsiteSpec extends Specification {
|
|
356
350
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
357
351
|
}
|
358
352
|
|
359
|
-
"not be applied if the glob doesn't match" in new EmptySite with VerboseLogger with MockAWS {
|
353
|
+
"not be applied if the glob doesn't match" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
360
354
|
config = """
|
361
355
|
|max_age:
|
362
356
|
| "*.js": 90
|
@@ -366,7 +360,7 @@ class S3WebsiteSpec extends Specification {
|
|
366
360
|
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
367
361
|
}
|
368
362
|
|
369
|
-
"be used to disable caching" in new EmptySite with VerboseLogger with MockAWS {
|
363
|
+
"be used to disable caching" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
370
364
|
config = "max_age: 0"
|
371
365
|
setLocalFile("index.html")
|
372
366
|
Push.pushSite
|
@@ -375,7 +369,7 @@ class S3WebsiteSpec extends Specification {
|
|
375
369
|
}
|
376
370
|
|
377
371
|
"max-age in config" should {
|
378
|
-
"respect the more specific glob" in new EmptySite with VerboseLogger with MockAWS {
|
372
|
+
"respect the more specific glob" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
379
373
|
config = """
|
380
374
|
|max_age:
|
381
375
|
| "assets/*": 150
|
@@ -389,7 +383,7 @@ class S3WebsiteSpec extends Specification {
|
|
389
383
|
}
|
390
384
|
|
391
385
|
"s3_reduced_redundancy: true in config" should {
|
392
|
-
"result in uploads being marked with reduced redundancy" in new EmptySite with VerboseLogger with MockAWS {
|
386
|
+
"result in uploads being marked with reduced redundancy" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
393
387
|
config = "s3_reduced_redundancy: true"
|
394
388
|
setLocalFile("file.exe")
|
395
389
|
Push.pushSite
|
@@ -398,7 +392,7 @@ class S3WebsiteSpec extends Specification {
|
|
398
392
|
}
|
399
393
|
|
400
394
|
"s3_reduced_redundancy: false in config" should {
|
401
|
-
"result in uploads being marked with the default storage class" in new EmptySite with VerboseLogger with MockAWS {
|
395
|
+
"result in uploads being marked with the default storage class" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
402
396
|
config = "s3_reduced_redundancy: false"
|
403
397
|
setLocalFile("file.exe")
|
404
398
|
Push.pushSite
|
@@ -407,7 +401,7 @@ class S3WebsiteSpec extends Specification {
|
|
407
401
|
}
|
408
402
|
|
409
403
|
"redirect in config" should {
|
410
|
-
"result in a redirect instruction that is sent to AWS" in new EmptySite with VerboseLogger with MockAWS {
|
404
|
+
"result in a redirect instruction that is sent to AWS" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
411
405
|
config = """
|
412
406
|
|redirects:
|
413
407
|
| index.php: /index.html
|
@@ -416,7 +410,7 @@ class S3WebsiteSpec extends Specification {
|
|
416
410
|
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
417
411
|
}
|
418
412
|
|
419
|
-
"result in max-age=0 Cache-Control header on the object" in new EmptySite with VerboseLogger with MockAWS {
|
413
|
+
"result in max-age=0 Cache-Control header on the object" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
420
414
|
config = """
|
421
415
|
|redirects:
|
422
416
|
| index.php: /index.html
|
@@ -427,7 +421,7 @@ class S3WebsiteSpec extends Specification {
|
|
427
421
|
}
|
428
422
|
|
429
423
|
"redirect in config and an object on the S3 bucket" should {
|
430
|
-
"not result in the S3 object being deleted" in new EmptySite with VerboseLogger with MockAWS {
|
424
|
+
"not result in the S3 object being deleted" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
431
425
|
config = """
|
432
426
|
|redirects:
|
433
427
|
| index.php: /index.html
|
@@ -440,7 +434,7 @@ class S3WebsiteSpec extends Specification {
|
|
440
434
|
}
|
441
435
|
|
442
436
|
"dotfiles" should {
|
443
|
-
"be included in the pushed files" in new EmptySite with VerboseLogger with MockAWS {
|
437
|
+
"be included in the pushed files" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
444
438
|
setLocalFile(".vimrc")
|
445
439
|
Push.pushSite
|
446
440
|
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
@@ -448,25 +442,25 @@ class S3WebsiteSpec extends Specification {
|
|
448
442
|
}
|
449
443
|
|
450
444
|
"content type inference" should {
|
451
|
-
"add charset=utf-8 to all html documents" in new EmptySite with VerboseLogger with MockAWS {
|
445
|
+
"add charset=utf-8 to all html documents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
452
446
|
setLocalFile("index.html")
|
453
447
|
Push.pushSite
|
454
448
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
455
449
|
}
|
456
450
|
|
457
|
-
"add charset=utf-8 to all text documents" in new EmptySite with VerboseLogger with MockAWS {
|
451
|
+
"add charset=utf-8 to all text documents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
458
452
|
setLocalFile("index.txt")
|
459
453
|
Push.pushSite
|
460
454
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/plain; charset=utf-8")
|
461
455
|
}
|
462
456
|
|
463
|
-
"add charset=utf-8 to all json documents" in new EmptySite with VerboseLogger with MockAWS {
|
457
|
+
"add charset=utf-8 to all json documents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
464
458
|
setLocalFile("data.json")
|
465
459
|
Push.pushSite
|
466
460
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("application/json; charset=utf-8")
|
467
461
|
}
|
468
462
|
|
469
|
-
"resolve the content type from file contents" in new EmptySite with VerboseLogger with MockAWS {
|
463
|
+
"resolve the content type from file contents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
470
464
|
setLocalFileWithContent(("index", "<html><body><h1>hi</h1></body></html>"))
|
471
465
|
Push.pushSite
|
472
466
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
@@ -474,7 +468,7 @@ class S3WebsiteSpec extends Specification {
|
|
474
468
|
}
|
475
469
|
|
476
470
|
"ERB in config file" should {
|
477
|
-
"be evaluated" in new EmptySite with VerboseLogger with MockAWS {
|
471
|
+
"be evaluated" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
478
472
|
config = """
|
479
473
|
|redirects:
|
480
474
|
|<%= ('a'..'f').to_a.map do |t| ' '+t+ ': /'+t+'.html' end.join('\n')%>
|
@@ -486,17 +480,67 @@ class S3WebsiteSpec extends Specification {
|
|
486
480
|
}
|
487
481
|
|
488
482
|
"logging" should {
|
489
|
-
"print the debug messages when --verbose is defined" in new EmptySite with VerboseLogger with MockAWS {
|
483
|
+
"print the debug messages when --verbose is defined" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
490
484
|
Push.pushSite
|
491
485
|
logEntries must contain("[debg] Querying S3 files")
|
492
486
|
}
|
493
487
|
|
494
|
-
"not print the debug messages by default" in new EmptySite with NonVerboseLogger with MockAWS {
|
488
|
+
"not print the debug messages by default" in new EmptySite with NonVerboseLogger with MockAWS with DefaultRunMode {
|
495
489
|
Push.pushSite
|
496
490
|
logEntries.forall(_.contains("[debg]")) must beFalse
|
497
491
|
}
|
498
492
|
}
|
499
493
|
|
494
|
+
"dry run" should {
|
495
|
+
"not push updates" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
496
|
+
setLocalFileWithContent(("index.html", "<div>new</div>"))
|
497
|
+
setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
|
498
|
+
Push.pushSite
|
499
|
+
noUploadsOccurred must beTrue
|
500
|
+
}
|
501
|
+
|
502
|
+
"not push redirects" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
503
|
+
config =
|
504
|
+
"""
|
505
|
+
|redirects:
|
506
|
+
| index.php: /index.html
|
507
|
+
""".stripMargin
|
508
|
+
Push.pushSite
|
509
|
+
noUploadsOccurred must beTrue
|
510
|
+
}
|
511
|
+
|
512
|
+
"not push deletes" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
513
|
+
setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
|
514
|
+
Push.pushSite
|
515
|
+
noUploadsOccurred must beTrue
|
516
|
+
}
|
517
|
+
|
518
|
+
"not push new files" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
519
|
+
setLocalFile("index.html")
|
520
|
+
Push.pushSite
|
521
|
+
noUploadsOccurred must beTrue
|
522
|
+
}
|
523
|
+
|
524
|
+
"not invalidate files" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
525
|
+
config = "cloudfront_invalidation_id: AABBCC"
|
526
|
+
setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
|
527
|
+
Push.pushSite
|
528
|
+
noInvalidationsOccurred must beTrue
|
529
|
+
}
|
530
|
+
}
|
531
|
+
|
532
|
+
trait DefaultRunMode {
|
533
|
+
implicit def pushMode: PushMode = new PushMode {
|
534
|
+
def dryRun = false
|
535
|
+
}
|
536
|
+
}
|
537
|
+
|
538
|
+
trait DryRunMode {
|
539
|
+
implicit def pushMode: PushMode = new PushMode {
|
540
|
+
def dryRun = true
|
541
|
+
}
|
542
|
+
}
|
543
|
+
|
500
544
|
trait MockAWS extends MockS3 with MockCloudFront with Scope
|
501
545
|
|
502
546
|
trait VerboseLogger extends LogCapturer {
|
@@ -518,7 +562,7 @@ class S3WebsiteSpec extends Specification {
|
|
518
562
|
|
519
563
|
trait MockCloudFront extends MockAWSHelper {
|
520
564
|
val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
|
521
|
-
implicit val cfSettings:
|
565
|
+
implicit val cfSettings: CloudFrontSetting = CloudFrontSetting(
|
522
566
|
cfClient = _ => amazonCloudFrontClient,
|
523
567
|
retryTimeUnit = MICROSECONDS
|
524
568
|
)
|
@@ -564,7 +608,7 @@ class S3WebsiteSpec extends Specification {
|
|
564
608
|
|
565
609
|
trait MockS3 extends MockAWSHelper {
|
566
610
|
val amazonS3Client = mock(classOf[AmazonS3])
|
567
|
-
implicit val s3Settings:
|
611
|
+
implicit val s3Settings: S3Setting = S3Setting(
|
568
612
|
s3Client = _ => amazonS3Client,
|
569
613
|
retryTimeUnit = MICROSECONDS
|
570
614
|
)
|
@@ -590,8 +634,6 @@ class S3WebsiteSpec extends Specification {
|
|
590
634
|
}
|
591
635
|
}
|
592
636
|
|
593
|
-
val s3 = new S3()
|
594
|
-
|
595
637
|
def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
596
638
|
doAnswer(temporaryFailure(classOf[PutObjectResult]))
|
597
639
|
.when(amazonS3Client)
|
@@ -611,7 +653,6 @@ class S3WebsiteSpec extends Specification {
|
|
611
653
|
}
|
612
654
|
|
613
655
|
def asSeenByS3Client(upload: Upload)(implicit config: Config, logger: Logger): PutObjectRequest = {
|
614
|
-
Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
|
615
656
|
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
616
657
|
verify(amazonS3Client).putObject(req.capture())
|
617
658
|
req.getValue
|