s3_website_monadic 0.0.27 → 0.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +10 -0
- data/bin/s3_website_monadic +13 -1
- data/lib/s3_website/version.rb +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +10 -10
- data/src/main/scala/s3/website/Push.scala +31 -21
- data/src/main/scala/s3/website/S3.scala +16 -15
- data/src/main/scala/s3/website/Utils.scala +30 -5
- data/src/main/scala/s3/website/model/push.scala +0 -3
- data/src/main/scala/s3/website/package.scala +37 -3
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +106 -65
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7900f28823f8541d10361f8b99020be7a403bb70
|
4
|
+
data.tar.gz: 2e0dad3c0abf21757fce241e11a7145f58d44cd4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 38a3e67004aeeb887118d7f8bfa7f5fef076bbc1c8d72ce6d6aaf409f51ed279c6602b51240ab7532bd45bf961189d2a75c1f25949e28d782b151d7b1cea8903
|
7
|
+
data.tar.gz: a1e09461dec20cec96591c7b67fe6ff4299a474e30b339972cedf49408e7629028e11ae0cb9ab5e7e67a12fdbd04f03af3f2e563ffa9a1a2ac739abd99bfa7e5
|
data/README.md
CHANGED
@@ -335,6 +335,16 @@ If you experience the "too many open files" error, either increase the amount of
|
|
335
335
|
maximum open files (on Unix-like systems, see `man ulimit`) or decrease the
|
336
336
|
`concurrency_level` setting.
|
337
337
|
|
338
|
+
## Simulating deployments
|
339
|
+
|
340
|
+
You can simulate the `s3_website_monadic push` operation by adding the
|
341
|
+
`--dry-run` switch. The dry run mode will not apply any modifications on your S3
|
342
|
+
bucket or CloudFront distribution. It will merely print out what the `push`
|
343
|
+
operation would actually do if run without the dry switch.
|
344
|
+
|
345
|
+
You can use the dry run mode if you are unsure what kind of effects the `push`
|
346
|
+
operation would cause to your live website.
|
347
|
+
|
338
348
|
## Example configurations
|
339
349
|
|
340
350
|
See
|
data/bin/s3_website_monadic
CHANGED
@@ -59,6 +59,12 @@ class Cli < Thor
|
|
59
59
|
:default => false,
|
60
60
|
:desc => "Print verbose output"
|
61
61
|
)
|
62
|
+
option(
|
63
|
+
:dry_run,
|
64
|
+
:type => :boolean,
|
65
|
+
:default => false,
|
66
|
+
:desc => "Run the operation without actually making the modifications. When this switch is on, changes will not be applied on the S3 website. They will be only printed to the console."
|
67
|
+
)
|
62
68
|
desc 'push', 'Push local files with the S3 website'
|
63
69
|
long_desc <<-LONGDESC
|
64
70
|
`s3_website push` will upload new and changes files to S3. It will
|
@@ -82,7 +88,13 @@ end
|
|
82
88
|
def run_s3_website_jar(jar_file, call_dir, logger)
|
83
89
|
site_path = File.expand_path(S3Website::Paths.infer_site_path options[:site], call_dir)
|
84
90
|
config_dir = File.expand_path options[:config_dir]
|
85
|
-
|
91
|
+
|
92
|
+
args = [
|
93
|
+
"--site=#{site_path}",
|
94
|
+
"--config-dir=#{config_dir}",
|
95
|
+
"#{'--verbose' if options[:verbose]}",
|
96
|
+
"#{'--dry-run' if options[:dry_run]}"
|
97
|
+
].join ' '
|
86
98
|
logger.debug_msg "Using #{jar_file}"
|
87
99
|
if system("java -cp #{jar_file} s3.website.Push #{args}")
|
88
100
|
exit 0
|
data/lib/s3_website/version.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
package s3.website
|
2
2
|
|
3
|
-
import s3.website.model.{Update,
|
3
|
+
import s3.website.model.{Update, Config}
|
4
4
|
import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
|
5
|
-
import s3.website.CloudFront.{
|
5
|
+
import s3.website.CloudFront.{CloudFrontSetting, SuccessfulInvalidation, FailedInvalidation}
|
6
6
|
import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
|
7
7
|
import scala.collection.JavaConversions._
|
8
8
|
import scala.concurrent.duration._
|
@@ -12,15 +12,15 @@ import java.net.URI
|
|
12
12
|
import Utils._
|
13
13
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
14
14
|
|
15
|
-
class CloudFront(implicit cloudFrontSettings:
|
15
|
+
class CloudFront(implicit cloudFrontSettings: CloudFrontSetting, config: Config, logger: Logger, pushMode: PushMode) {
|
16
16
|
val cloudFront = cloudFrontSettings.cfClient(config)
|
17
17
|
|
18
18
|
def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
|
19
19
|
(implicit ec: ExecutionContextExecutor): InvalidationResult =
|
20
20
|
Future {
|
21
|
-
|
22
|
-
cloudFront.createInvalidation(invalidationReq)
|
21
|
+
if (!pushMode.dryRun) cloudFront createInvalidation new CreateInvalidationRequest(distributionId, invalidationBatch)
|
23
22
|
val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
|
23
|
+
logger.debug(invalidationBatch.getPaths.getItems.map(item => s"${Invalidated.renderVerb} $item") mkString "\n")
|
24
24
|
logger.info(result)
|
25
25
|
Right(result)
|
26
26
|
} recoverWith (tooManyInvalidationsRetry(invalidationBatch, distributionId, attempt) orElse retry(attempt)(
|
@@ -44,7 +44,7 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config
|
|
44
44
|
val basicInfo = s"The maximum amount of CloudFront invalidations has exceeded. Trying again in $sleepDuration, please wait."
|
45
45
|
val extendedInfo =
|
46
46
|
s"""|$basicInfo
|
47
|
-
|
|
47
|
+
|For more information, see http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/Invalidation.html#InvalidationLimits"""
|
48
48
|
.stripMargin
|
49
49
|
if (attempt == 1)
|
50
50
|
extendedInfo
|
@@ -59,8 +59,8 @@ object CloudFront {
|
|
59
59
|
|
60
60
|
type CloudFrontClientProvider = (Config) => AmazonCloudFront
|
61
61
|
|
62
|
-
case class SuccessfulInvalidation(invalidatedItemsCount: Int) extends SuccessReport {
|
63
|
-
def reportMessage = s"Invalidated ${invalidatedItemsCount ofType "item"} on CloudFront"
|
62
|
+
case class SuccessfulInvalidation(invalidatedItemsCount: Int)(implicit pushMode: PushMode) extends SuccessReport {
|
63
|
+
def reportMessage = s"${Invalidated.renderVerb} ${invalidatedItemsCount ofType "item"} on CloudFront"
|
64
64
|
}
|
65
65
|
|
66
66
|
case class FailedInvalidation(error: Throwable) extends FailureReport {
|
@@ -129,8 +129,8 @@ object CloudFront {
|
|
129
129
|
case _ => false
|
130
130
|
}
|
131
131
|
|
132
|
-
case class
|
132
|
+
case class CloudFrontSetting(
|
133
133
|
cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
|
134
134
|
retryTimeUnit: TimeUnit = MINUTES
|
135
|
-
) extends
|
135
|
+
) extends RetrySetting
|
136
136
|
}
|
@@ -20,21 +20,23 @@ import scala.collection.mutable.ArrayBuffer
|
|
20
20
|
import s3.website.CloudFront._
|
21
21
|
import s3.website.S3.SuccessfulDelete
|
22
22
|
import s3.website.CloudFront.SuccessfulInvalidation
|
23
|
-
import s3.website.S3.
|
24
|
-
import s3.website.CloudFront.
|
23
|
+
import s3.website.S3.S3Setting
|
24
|
+
import s3.website.CloudFront.CloudFrontSetting
|
25
25
|
import s3.website.S3.SuccessfulUpload
|
26
26
|
import s3.website.CloudFront.FailedInvalidation
|
27
|
+
import scala.Int
|
27
28
|
|
28
29
|
object Push {
|
29
30
|
|
30
31
|
def pushSite(
|
31
32
|
implicit site: Site,
|
32
33
|
executor: ExecutionContextExecutor,
|
33
|
-
s3Settings:
|
34
|
-
cloudFrontSettings:
|
35
|
-
logger: Logger
|
34
|
+
s3Settings: S3Setting,
|
35
|
+
cloudFrontSettings: CloudFrontSetting,
|
36
|
+
logger: Logger,
|
37
|
+
pushMode: PushMode
|
36
38
|
): ExitCode = {
|
37
|
-
logger.info(s"
|
39
|
+
logger.info(s"${Deploy.renderVerb} ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
38
40
|
val utils = new Utils
|
39
41
|
|
40
42
|
val redirects = Redirect.resolveRedirects
|
@@ -63,7 +65,8 @@ object Push {
|
|
63
65
|
|
64
66
|
def invalidateCloudFrontItems
|
65
67
|
(errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
|
66
|
-
(implicit config: Config, cloudFrontSettings:
|
68
|
+
(implicit config: Config, cloudFrontSettings: CloudFrontSetting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
|
69
|
+
Option[InvalidationSucceeded] = {
|
67
70
|
config.cloudfront_distribution_id.map {
|
68
71
|
distributionId =>
|
69
72
|
val pushSuccessReports = errorsOrFinishedPushOps.fold(
|
@@ -101,7 +104,7 @@ object Push {
|
|
101
104
|
type InvalidationSucceeded = Boolean
|
102
105
|
|
103
106
|
def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])
|
104
|
-
(implicit config: Config, logger: Logger): ExitCode = {
|
107
|
+
(implicit config: Config, logger: Logger, pushMode: PushMode): ExitCode = {
|
105
108
|
errorsOrFinishedUploads.right.foreach { finishedUploads =>
|
106
109
|
val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
|
107
110
|
logger.info(s"Summary: $pushCounts")
|
@@ -120,10 +123,13 @@ object Push {
|
|
120
123
|
if (allInvalidationsSucceeded) 0 else 1
|
121
124
|
)
|
122
125
|
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
126
|
+
exitCode match {
|
127
|
+
case 0 if !pushMode.dryRun =>
|
128
|
+
logger.info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
129
|
+
case 1 =>
|
130
|
+
logger.fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
131
|
+
case _ =>
|
132
|
+
}
|
127
133
|
exitCode
|
128
134
|
}
|
129
135
|
|
@@ -149,18 +155,18 @@ object Push {
|
|
149
155
|
)
|
150
156
|
}
|
151
157
|
|
152
|
-
def pushCountsToString(pushCounts: PushCounts): String =
|
158
|
+
def pushCountsToString(pushCounts: PushCounts)(implicit pushMode: PushMode): String =
|
153
159
|
pushCounts match {
|
154
160
|
case PushCounts(updates, newFiles, failures, redirects, deletes)
|
155
161
|
if updates == 0 && newFiles == 0 && failures == 0 && redirects == 0 && deletes == 0 =>
|
156
|
-
|
162
|
+
PushNothing.renderVerb
|
157
163
|
case PushCounts(updates, newFiles, failures, redirects, deletes) =>
|
158
164
|
val reportClauses: scala.collection.mutable.ArrayBuffer[String] = ArrayBuffer()
|
159
|
-
if (updates > 0) reportClauses += s"Updated ${updates ofType "file"}."
|
160
|
-
if (newFiles > 0) reportClauses += s"Created ${newFiles ofType "file"}."
|
165
|
+
if (updates > 0) reportClauses += s"${Updated.renderVerb} ${updates ofType "file"}."
|
166
|
+
if (newFiles > 0) reportClauses += s"${Created.renderVerb} ${newFiles ofType "file"}."
|
161
167
|
if (failures > 0) reportClauses += s"${failures ofType "operation"} failed." // This includes both failed uploads and deletes.
|
162
|
-
if (redirects > 0) reportClauses += s"Applied ${redirects ofType "redirect"}."
|
163
|
-
if (deletes > 0) reportClauses += s"Deleted ${deletes ofType "file"}."
|
168
|
+
if (redirects > 0) reportClauses += s"${Applied.renderVerb} ${redirects ofType "redirect"}."
|
169
|
+
if (deletes > 0) reportClauses += s"${Deleted.renderVerb} ${deletes ofType "file"}."
|
164
170
|
reportClauses.mkString(" ")
|
165
171
|
}
|
166
172
|
|
@@ -182,20 +188,24 @@ object Push {
|
|
182
188
|
@Option def site: String
|
183
189
|
@Option(longName = Array("config-dir")) def configDir: String
|
184
190
|
@Option def verbose: Boolean
|
191
|
+
@Option(longName = Array("dry-run")) def dryRun: Boolean
|
185
192
|
}
|
186
193
|
|
187
194
|
def main(args: Array[String]) {
|
188
195
|
val cliArgs = CliFactory.parseArguments(classOf[CliArgs], args:_*)
|
189
|
-
implicit val s3Settings =
|
190
|
-
implicit val cloudFrontSettings =
|
196
|
+
implicit val s3Settings = S3Setting()
|
197
|
+
implicit val cloudFrontSettings = CloudFrontSetting()
|
191
198
|
implicit val logger: Logger = new Logger(cliArgs.verbose)
|
199
|
+
implicit val pushMode = new PushMode {
|
200
|
+
def dryRun = cliArgs.dryRun
|
201
|
+
}
|
192
202
|
val errorOrPushStatus = push(siteInDirectory = cliArgs.site, withConfigDirectory = cliArgs.configDir)
|
193
203
|
errorOrPushStatus.left foreach (err => logger.fail(s"Could not load the site: ${err.reportMessage}"))
|
194
204
|
System exit errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus)
|
195
205
|
}
|
196
206
|
|
197
207
|
def push(siteInDirectory: String, withConfigDirectory: String)
|
198
|
-
(implicit s3Settings:
|
208
|
+
(implicit s3Settings: S3Setting, cloudFrontSettings: CloudFrontSetting, logger: Logger, pushMode: PushMode) =
|
199
209
|
loadSite(withConfigDirectory + "/s3_website.yml", siteInDirectory)
|
200
210
|
.right
|
201
211
|
.map {
|
@@ -14,15 +14,15 @@ import s3.website.S3.SuccessfulDelete
|
|
14
14
|
import s3.website.S3.FailedUpload
|
15
15
|
import scala.Some
|
16
16
|
import s3.website.S3.FailedDelete
|
17
|
-
import s3.website.S3.
|
17
|
+
import s3.website.S3.S3Setting
|
18
18
|
|
19
|
-
class S3(implicit s3Settings:
|
19
|
+
class S3(implicit s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor) {
|
20
20
|
|
21
21
|
def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)
|
22
22
|
(implicit config: Config, logger: Logger): Future[Either[FailedUpload, SuccessfulUpload]] =
|
23
23
|
Future {
|
24
24
|
val putObjectRequest = toPutObjectRequest(upload)
|
25
|
-
s3Settings.s3Client(config) putObject putObjectRequest
|
25
|
+
if (!pushMode.dryRun) s3Settings.s3Client(config) putObject putObjectRequest
|
26
26
|
val report = SuccessfulUpload(upload, putObjectRequest)
|
27
27
|
logger.info(report)
|
28
28
|
Right(report)
|
@@ -34,7 +34,7 @@ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
|
|
34
34
|
def delete(s3Key: String, a: Attempt = 1)
|
35
35
|
(implicit config: Config, logger: Logger): Future[Either[FailedDelete, SuccessfulDelete]] =
|
36
36
|
Future {
|
37
|
-
s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
37
|
+
if (!pushMode.dryRun) s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
38
38
|
val report = SuccessfulDelete(s3Key)
|
39
39
|
logger.info(report)
|
40
40
|
Right(report)
|
@@ -84,7 +84,7 @@ object S3 {
|
|
84
84
|
|
85
85
|
def resolveS3FilesAndUpdates(localFiles: Seq[LocalFile])
|
86
86
|
(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil, attempt: Attempt = 1, onFlightUpdateFutures: UpdateFutures = Nil)
|
87
|
-
(implicit config: Config, s3Settings:
|
87
|
+
(implicit config: Config, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):
|
88
88
|
ErrorOrS3FilesAndUpdates = Future {
|
89
89
|
logger.debug(nextMarker.fold
|
90
90
|
("Querying S3 files")
|
@@ -97,12 +97,12 @@ object S3 {
|
|
97
97
|
req
|
98
98
|
})
|
99
99
|
val summaryIndex = objects.getObjectSummaries.map { summary => (summary.getETag, summary.getKey) }.toSet // Index to avoid O(n^2) lookups
|
100
|
-
def
|
100
|
+
def shouldUpdate(lf: LocalFile) =
|
101
101
|
summaryIndex.exists((md5AndS3Key) =>
|
102
102
|
md5AndS3Key._1 != lf.md5 && md5AndS3Key._2 == lf.s3Key
|
103
103
|
)
|
104
104
|
val updateFutures: UpdateFutures = localFiles.collect {
|
105
|
-
case lf: LocalFile if
|
105
|
+
case lf: LocalFile if shouldUpdate(lf) =>
|
106
106
|
val errorOrUpdate = LocalFile
|
107
107
|
.toUpload(lf)
|
108
108
|
.right
|
@@ -138,7 +138,8 @@ object S3 {
|
|
138
138
|
def s3Key: String
|
139
139
|
}
|
140
140
|
|
141
|
-
case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest)
|
141
|
+
case class SuccessfulUpload(upload: Upload with UploadTypeResolved, putObjectRequest: PutObjectRequest)
|
142
|
+
(implicit pushMode: PushMode) extends PushSuccessReport {
|
142
143
|
val metadata = putObjectRequest.getMetadata
|
143
144
|
def metadataReport =
|
144
145
|
(metadata.getCacheControl :: metadata.getContentType :: metadata.getContentEncoding :: putObjectRequest.getStorageClass :: Nil)
|
@@ -147,16 +148,16 @@ object S3 {
|
|
147
148
|
|
148
149
|
def reportMessage =
|
149
150
|
upload.uploadType match {
|
150
|
-
case NewFile => s"Created ${upload.s3Key} ($metadataReport)"
|
151
|
-
case Update => s"Updated ${upload.s3Key} ($metadataReport)"
|
152
|
-
case Redirect => s"Redirected ${upload.essence.left.get.key} to ${upload.essence.left.get.redirectTarget}"
|
151
|
+
case NewFile => s"${Created.renderVerb} ${upload.s3Key} ($metadataReport)"
|
152
|
+
case Update => s"${Updated.renderVerb} ${upload.s3Key} ($metadataReport)"
|
153
|
+
case Redirect => s"${Redirected.renderVerb} ${upload.essence.left.get.key} to ${upload.essence.left.get.redirectTarget}"
|
153
154
|
}
|
154
155
|
|
155
156
|
def s3Key = upload.s3Key
|
156
157
|
}
|
157
158
|
|
158
|
-
case class SuccessfulDelete(s3Key: String) extends PushSuccessReport {
|
159
|
-
def reportMessage = s"Deleted $s3Key"
|
159
|
+
case class SuccessfulDelete(s3Key: String)(implicit pushMode: PushMode) extends PushSuccessReport {
|
160
|
+
def reportMessage = s"${Deleted.renderVerb} $s3Key"
|
160
161
|
}
|
161
162
|
|
162
163
|
case class FailedUpload(s3Key: String, error: Throwable) extends PushFailureReport {
|
@@ -169,8 +170,8 @@ object S3 {
|
|
169
170
|
|
170
171
|
type S3ClientProvider = (Config) => AmazonS3
|
171
172
|
|
172
|
-
case class
|
173
|
+
case class S3Setting(
|
173
174
|
s3Client: S3ClientProvider = S3.awsS3Client,
|
174
175
|
retryTimeUnit: TimeUnit = SECONDS
|
175
|
-
) extends
|
176
|
+
) extends RetrySetting
|
176
177
|
}
|
@@ -18,14 +18,39 @@ object Utils {
|
|
18
18
|
|
19
19
|
class Logger(verboseOutput: Boolean, logMessage: (String) => Unit = println) {
|
20
20
|
import Rainbow._
|
21
|
-
def debug(msg: String) = if (verboseOutput)
|
22
|
-
def info(msg: String) =
|
23
|
-
def fail(msg: String) =
|
21
|
+
def debug(msg: String) = if (verboseOutput) log(Debug, msg)
|
22
|
+
def info(msg: String) = log(Info, msg)
|
23
|
+
def fail(msg: String) = log(Failure, msg)
|
24
24
|
|
25
|
-
def info(report: SuccessReport) =
|
25
|
+
def info(report: SuccessReport) = log(Success, report.reportMessage)
|
26
26
|
def info(report: FailureReport) = fail(report.reportMessage)
|
27
27
|
|
28
|
-
def pending(msg: String) =
|
28
|
+
def pending(msg: String) = log(Wait, msg)
|
29
|
+
|
30
|
+
private def log(logType: LogType, msgRaw: String) {
|
31
|
+
val msg = msgRaw.replaceAll("\\n", "\n ") // Indent new lines, so that they arrange nicely with other log lines
|
32
|
+
logMessage(s"[$logType] $msg")
|
33
|
+
}
|
34
|
+
|
35
|
+
sealed trait LogType {
|
36
|
+
val prefix: String
|
37
|
+
override def toString = prefix
|
38
|
+
}
|
39
|
+
case object Debug extends LogType {
|
40
|
+
val prefix = "debg".cyan
|
41
|
+
}
|
42
|
+
case object Info extends LogType {
|
43
|
+
val prefix = "info".blue
|
44
|
+
}
|
45
|
+
case object Success extends LogType {
|
46
|
+
val prefix = "succ".green
|
47
|
+
}
|
48
|
+
case object Failure extends LogType {
|
49
|
+
val prefix = "fail".red
|
50
|
+
}
|
51
|
+
case object Wait extends LogType {
|
52
|
+
val prefix = "wait".yellow
|
53
|
+
}
|
29
54
|
}
|
30
55
|
|
31
56
|
/**
|
@@ -17,9 +17,43 @@ package object website {
|
|
17
17
|
|
18
18
|
trait ErrorReport extends Report
|
19
19
|
|
20
|
-
trait
|
20
|
+
trait RetrySetting {
|
21
21
|
def retryTimeUnit: TimeUnit
|
22
22
|
}
|
23
|
+
|
24
|
+
trait PushMode {
|
25
|
+
def dryRun: Boolean
|
26
|
+
}
|
27
|
+
|
28
|
+
trait PushAction {
|
29
|
+
def actionName = getClass.getSimpleName.replace("$", "") // case object class names contain the '$' char
|
30
|
+
|
31
|
+
def renderVerb(implicit pushMode: PushMode): String =
|
32
|
+
if (pushMode.dryRun)
|
33
|
+
s"Would have ${actionName.toLowerCase}"
|
34
|
+
else
|
35
|
+
s"$actionName"
|
36
|
+
}
|
37
|
+
case object Created extends PushAction
|
38
|
+
case object Updated extends PushAction
|
39
|
+
case object Redirected extends PushAction
|
40
|
+
case object Deleted extends PushAction
|
41
|
+
case object Invalidated extends PushAction
|
42
|
+
case object Applied extends PushAction
|
43
|
+
case object PushNothing extends PushAction {
|
44
|
+
override def renderVerb(implicit pushMode: PushMode) =
|
45
|
+
if (pushMode.dryRun)
|
46
|
+
s"Would have pushed nothing"
|
47
|
+
else
|
48
|
+
s"There was nothing to push"
|
49
|
+
}
|
50
|
+
case object Deploy extends PushAction {
|
51
|
+
override def renderVerb(implicit pushMode: PushMode) =
|
52
|
+
if (pushMode.dryRun)
|
53
|
+
s"Simulating the deployment of"
|
54
|
+
else
|
55
|
+
s"Deploying"
|
56
|
+
}
|
23
57
|
|
24
58
|
type PushErrorOrSuccess = Either[PushFailureReport, PushSuccessReport]
|
25
59
|
|
@@ -27,7 +61,7 @@ package object website {
|
|
27
61
|
|
28
62
|
def retry[L <: Report, R](attempt: Attempt)
|
29
63
|
(createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
|
30
|
-
(implicit
|
64
|
+
(implicit retrySetting: RetrySetting, ec: ExecutionContextExecutor, logger: Logger):
|
31
65
|
PartialFunction[Throwable, Future[Either[L, R]]] = {
|
32
66
|
case error: Throwable if attempt == 6 || isIrrecoverable(error) =>
|
33
67
|
val failureReport = createFailureReport(error)
|
@@ -35,7 +69,7 @@ package object website {
|
|
35
69
|
Future(Left(failureReport))
|
36
70
|
case error: Throwable =>
|
37
71
|
val failureReport = createFailureReport(error)
|
38
|
-
val sleepDuration = Duration(fibs.drop(attempt + 1).head,
|
72
|
+
val sleepDuration = Duration(fibs.drop(attempt + 1).head, retrySetting.retryTimeUnit)
|
39
73
|
logger.pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
40
74
|
Thread.sleep(sleepDuration.toMillis)
|
41
75
|
retryAction(attempt + 1)
|
@@ -11,14 +11,12 @@ import org.mockito.Mockito._
|
|
11
11
|
import com.amazonaws.services.s3.AmazonS3
|
12
12
|
import com.amazonaws.services.s3.model._
|
13
13
|
import scala.concurrent.ExecutionContext.Implicits.global
|
14
|
-
import scala.concurrent.Await
|
15
14
|
import scala.concurrent.duration._
|
16
|
-
import s3.website.S3.
|
15
|
+
import s3.website.S3.S3Setting
|
17
16
|
import scala.collection.JavaConversions._
|
18
|
-
import s3.website.model.NewFile
|
19
17
|
import com.amazonaws.AmazonServiceException
|
20
18
|
import org.apache.commons.codec.digest.DigestUtils.md5Hex
|
21
|
-
import s3.website.CloudFront.
|
19
|
+
import s3.website.CloudFront.CloudFrontSetting
|
22
20
|
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
23
21
|
import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
|
24
22
|
import org.mockito.stubbing.Answer
|
@@ -30,7 +28,7 @@ import scala.collection.mutable
|
|
30
28
|
class S3WebsiteSpec extends Specification {
|
31
29
|
|
32
30
|
"gzip: true" should {
|
33
|
-
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS {
|
31
|
+
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
34
32
|
config = "gzip: true"
|
35
33
|
setLocalFileWithContent(("styles.css", "<h1>hi again</h1>"))
|
36
34
|
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
@@ -38,7 +36,7 @@ class S3WebsiteSpec extends Specification {
|
|
38
36
|
sentPutObjectRequest.getKey must equalTo("styles.css")
|
39
37
|
}
|
40
38
|
|
41
|
-
"not update a gzipped S3 object if the contents has not changed" in new EmptySite with VerboseLogger with MockAWS {
|
39
|
+
"not update a gzipped S3 object if the contents has not changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
42
40
|
config = "gzip: true"
|
43
41
|
setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
|
44
42
|
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
@@ -51,7 +49,7 @@ class S3WebsiteSpec extends Specification {
|
|
51
49
|
gzip:
|
52
50
|
- .xml
|
53
51
|
""" should {
|
54
|
-
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS {
|
52
|
+
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
55
53
|
config = """
|
56
54
|
|gzip:
|
57
55
|
| - .xml
|
@@ -64,40 +62,40 @@ class S3WebsiteSpec extends Specification {
|
|
64
62
|
}
|
65
63
|
|
66
64
|
"push" should {
|
67
|
-
"not upload a file if it has not changed" in new EmptySite with VerboseLogger with MockAWS {
|
65
|
+
"not upload a file if it has not changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
68
66
|
setLocalFileWithContent(("index.html", "<div>hello</div>"))
|
69
67
|
setS3Files(S3File("index.html", md5Hex("<div>hello</div>")))
|
70
68
|
Push.pushSite
|
71
69
|
noUploadsOccurred must beTrue
|
72
70
|
}
|
73
71
|
|
74
|
-
"update a file if it has changed" in new EmptySite with VerboseLogger with MockAWS {
|
72
|
+
"update a file if it has changed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
75
73
|
setLocalFileWithContent(("index.html", "<h1>old text</h1>"))
|
76
74
|
setS3Files(S3File("index.html", md5Hex("<h1>new text</h1>")))
|
77
75
|
Push.pushSite
|
78
76
|
sentPutObjectRequest.getKey must equalTo("index.html")
|
79
77
|
}
|
80
78
|
|
81
|
-
"create a file if does not exist on S3" in new EmptySite with VerboseLogger with MockAWS {
|
79
|
+
"create a file if does not exist on S3" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
82
80
|
setLocalFile("index.html")
|
83
81
|
Push.pushSite
|
84
82
|
sentPutObjectRequest.getKey must equalTo("index.html")
|
85
83
|
}
|
86
84
|
|
87
|
-
"delete files that are on S3 but not on local file system" in new EmptySite with VerboseLogger with MockAWS {
|
85
|
+
"delete files that are on S3 but not on local file system" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
88
86
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
89
87
|
Push.pushSite
|
90
88
|
sentDelete must equalTo("old.html")
|
91
89
|
}
|
92
90
|
|
93
|
-
"try again if the upload fails" in new EmptySite with VerboseLogger with MockAWS {
|
91
|
+
"try again if the upload fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
94
92
|
setLocalFile("index.html")
|
95
93
|
uploadFailsAndThenSucceeds(howManyFailures = 5)
|
96
94
|
Push.pushSite
|
97
95
|
verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
98
96
|
}
|
99
97
|
|
100
|
-
"not try again if the upload fails on because of invalid credentials" in new EmptySite with VerboseLogger with MockAWS {
|
98
|
+
"not try again if the upload fails on because of invalid credentials" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
101
99
|
setLocalFile("index.html")
|
102
100
|
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
|
103
101
|
val e = new AmazonServiceException("your credentials are incorrect")
|
@@ -108,14 +106,14 @@ class S3WebsiteSpec extends Specification {
|
|
108
106
|
verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
109
107
|
}
|
110
108
|
|
111
|
-
"try again if the delete fails" in new EmptySite with VerboseLogger with MockAWS {
|
109
|
+
"try again if the delete fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
112
110
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
113
111
|
deleteFailsAndThenSucceeds(howManyFailures = 5)
|
114
112
|
Push.pushSite
|
115
113
|
verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
|
116
114
|
}
|
117
115
|
|
118
|
-
"try again if the object listing fails" in new EmptySite with VerboseLogger with MockAWS {
|
116
|
+
"try again if the object listing fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
119
117
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
120
118
|
objectListingFailsAndThenSucceeds(howManyFailures = 5)
|
121
119
|
Push.pushSite
|
@@ -124,7 +122,7 @@ class S3WebsiteSpec extends Specification {
|
|
124
122
|
}
|
125
123
|
|
126
124
|
"push with CloudFront" should {
|
127
|
-
"invalidate the updated CloudFront items" in new EmptySite with VerboseLogger with MockAWS {
|
125
|
+
"invalidate the updated CloudFront items" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
128
126
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
129
127
|
setLocalFiles("css/test.css", "articles/index.html")
|
130
128
|
setOutdatedS3Keys("css/test.css", "articles/index.html")
|
@@ -132,14 +130,14 @@ class S3WebsiteSpec extends Specification {
|
|
132
130
|
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/css/test.css" :: "/articles/index.html" :: Nil).sorted)
|
133
131
|
}
|
134
132
|
|
135
|
-
"not send CloudFront invalidation requests on new objects" in new EmptySite with VerboseLogger with MockAWS {
|
133
|
+
"not send CloudFront invalidation requests on new objects" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
136
134
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
137
135
|
setLocalFile("newfile.js")
|
138
136
|
Push.pushSite
|
139
137
|
noInvalidationsOccurred must beTrue
|
140
138
|
}
|
141
139
|
|
142
|
-
"not send CloudFront invalidation requests on redirect objects" in new EmptySite with VerboseLogger with MockAWS {
|
140
|
+
"not send CloudFront invalidation requests on redirect objects" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
143
141
|
config = """
|
144
142
|
|cloudfront_distribution_id: EGM1J2JJX9Z
|
145
143
|
|redirects:
|
@@ -149,7 +147,7 @@ class S3WebsiteSpec extends Specification {
|
|
149
147
|
noInvalidationsOccurred must beTrue
|
150
148
|
}
|
151
149
|
|
152
|
-
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new EmptySite with VerboseLogger with MockAWS {
|
150
|
+
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
153
151
|
setTooManyInvalidationsInProgress(4)
|
154
152
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
155
153
|
setLocalFile("test.css")
|
@@ -158,7 +156,7 @@ class S3WebsiteSpec extends Specification {
|
|
158
156
|
sentInvalidationRequests.length must equalTo(4)
|
159
157
|
}
|
160
158
|
|
161
|
-
"retry if CloudFront is temporarily unreachable" in new EmptySite with VerboseLogger with MockAWS {
|
159
|
+
"retry if CloudFront is temporarily unreachable" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
162
160
|
invalidationsFailAndThenSucceed(5)
|
163
161
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
164
162
|
setLocalFile("test.css")
|
@@ -167,7 +165,7 @@ class S3WebsiteSpec extends Specification {
|
|
167
165
|
sentInvalidationRequests.length must equalTo(6)
|
168
166
|
}
|
169
167
|
|
170
|
-
"encode unsafe characters in the keys" in new EmptySite with VerboseLogger with MockAWS {
|
168
|
+
"encode unsafe characters in the keys" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
171
169
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
172
170
|
setLocalFile("articles/arnold's file.html")
|
173
171
|
setOutdatedS3Keys("articles/arnold's file.html")
|
@@ -175,11 +173,7 @@ class S3WebsiteSpec extends Specification {
|
|
175
173
|
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/articles/arnold's%20file.html" :: Nil).sorted)
|
176
174
|
}
|
177
175
|
|
178
|
-
|
179
|
-
* Because CloudFront supports Default Root Objects (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DefaultRootObject.html),
|
180
|
-
* we have to guess
|
181
|
-
*/
|
182
|
-
"invalidate the root object '/' if a top-level object is updated or deleted" in new EmptySite with VerboseLogger with MockAWS {
|
176
|
+
"invalidate the root object '/' if a top-level object is updated or deleted" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
183
177
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
184
178
|
setLocalFile("maybe-index.html")
|
185
179
|
setOutdatedS3Keys("maybe-index.html")
|
@@ -189,7 +183,7 @@ class S3WebsiteSpec extends Specification {
|
|
189
183
|
}
|
190
184
|
|
191
185
|
"cloudfront_invalidate_root: true" should {
|
192
|
-
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new EmptySite with VerboseLogger with MockAWS {
|
186
|
+
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
193
187
|
config = """
|
194
188
|
|cloudfront_distribution_id: EGM1J2JJX9Z
|
195
189
|
|cloudfront_invalidate_root: true
|
@@ -202,7 +196,7 @@ class S3WebsiteSpec extends Specification {
|
|
202
196
|
}
|
203
197
|
|
204
198
|
"a site with over 1000 items" should {
|
205
|
-
"split the CloudFront invalidation requests into batches of 1000 items" in new EmptySite with VerboseLogger with MockAWS {
|
199
|
+
"split the CloudFront invalidation requests into batches of 1000 items" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
206
200
|
val files = (1 to 1002).map { i => s"lots-of-files/file-$i"}
|
207
201
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
208
202
|
setLocalFiles(files:_*)
|
@@ -215,18 +209,18 @@ class S3WebsiteSpec extends Specification {
|
|
215
209
|
}
|
216
210
|
|
217
211
|
"push exit status" should {
|
218
|
-
"be 0 all uploads succeed" in new EmptySite with VerboseLogger with MockAWS {
|
212
|
+
"be 0 all uploads succeed" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
219
213
|
setLocalFiles("file.txt")
|
220
214
|
Push.pushSite must equalTo(0)
|
221
215
|
}
|
222
216
|
|
223
|
-
"be 1 if any of the uploads fails" in new EmptySite with VerboseLogger with MockAWS {
|
217
|
+
"be 1 if any of the uploads fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
224
218
|
setLocalFiles("file.txt")
|
225
219
|
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
226
220
|
Push.pushSite must equalTo(1)
|
227
221
|
}
|
228
222
|
|
229
|
-
"be 1 if any of the redirects fails" in new EmptySite with VerboseLogger with MockAWS {
|
223
|
+
"be 1 if any of the redirects fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
230
224
|
config = """
|
231
225
|
|redirects:
|
232
226
|
| index.php: /index.html
|
@@ -235,14 +229,14 @@ class S3WebsiteSpec extends Specification {
|
|
235
229
|
Push.pushSite must equalTo(1)
|
236
230
|
}
|
237
231
|
|
238
|
-
"be 0 if CloudFront invalidations and uploads succeed"in new EmptySite with VerboseLogger with MockAWS {
|
232
|
+
"be 0 if CloudFront invalidations and uploads succeed"in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
239
233
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
240
234
|
setLocalFile("test.css")
|
241
235
|
setOutdatedS3Keys("test.css")
|
242
236
|
Push.pushSite must equalTo(0)
|
243
237
|
}
|
244
238
|
|
245
|
-
"be 1 if CloudFront is unreachable or broken"in new EmptySite with VerboseLogger with MockAWS {
|
239
|
+
"be 1 if CloudFront is unreachable or broken"in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
246
240
|
setCloudFrontAsInternallyBroken()
|
247
241
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
248
242
|
setLocalFile("test.css")
|
@@ -250,19 +244,19 @@ class S3WebsiteSpec extends Specification {
|
|
250
244
|
Push.pushSite must equalTo(1)
|
251
245
|
}
|
252
246
|
|
253
|
-
"be 0 if upload retry succeeds" in new EmptySite with VerboseLogger with MockAWS {
|
247
|
+
"be 0 if upload retry succeeds" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
254
248
|
setLocalFile("index.html")
|
255
249
|
uploadFailsAndThenSucceeds(howManyFailures = 1)
|
256
250
|
Push.pushSite must equalTo(0)
|
257
251
|
}
|
258
252
|
|
259
|
-
"be 1 if delete retry fails" in new EmptySite with VerboseLogger with MockAWS {
|
253
|
+
"be 1 if delete retry fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
260
254
|
setLocalFile("index.html")
|
261
255
|
uploadFailsAndThenSucceeds(howManyFailures = 6)
|
262
256
|
Push.pushSite must equalTo(1)
|
263
257
|
}
|
264
258
|
|
265
|
-
"be 1 if an object listing fails" in new EmptySite with VerboseLogger with MockAWS {
|
259
|
+
"be 1 if an object listing fails" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
266
260
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
267
261
|
objectListingFailsAndThenSucceeds(howManyFailures = 6)
|
268
262
|
Push.pushSite must equalTo(1)
|
@@ -270,7 +264,7 @@ class S3WebsiteSpec extends Specification {
|
|
270
264
|
}
|
271
265
|
|
272
266
|
"s3_website.yml file" should {
|
273
|
-
"never be uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
267
|
+
"never be uploaded" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
274
268
|
setLocalFile("s3_website.yml")
|
275
269
|
Push.pushSite
|
276
270
|
noUploadsOccurred must beTrue
|
@@ -278,7 +272,7 @@ class S3WebsiteSpec extends Specification {
|
|
278
272
|
}
|
279
273
|
|
280
274
|
"exclude_from_upload: string" should {
|
281
|
-
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
275
|
+
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
282
276
|
config = "exclude_from_upload: .DS_.*?"
|
283
277
|
setLocalFile(".DS_Store")
|
284
278
|
Push.pushSite
|
@@ -291,7 +285,7 @@ class S3WebsiteSpec extends Specification {
|
|
291
285
|
- regex
|
292
286
|
- another_exclusion
|
293
287
|
""" should {
|
294
|
-
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
288
|
+
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
295
289
|
config = """
|
296
290
|
|exclude_from_upload:
|
297
291
|
| - .DS_.*?
|
@@ -304,7 +298,7 @@ class S3WebsiteSpec extends Specification {
|
|
304
298
|
}
|
305
299
|
|
306
300
|
"ignore_on_server: value" should {
|
307
|
-
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS {
|
301
|
+
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
308
302
|
config = "ignore_on_server: logs"
|
309
303
|
setS3Files(S3File("logs/log.txt", ""))
|
310
304
|
Push.pushSite
|
@@ -317,7 +311,7 @@ class S3WebsiteSpec extends Specification {
|
|
317
311
|
- regex
|
318
312
|
- another_ignore
|
319
313
|
""" should {
|
320
|
-
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS {
|
314
|
+
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
321
315
|
config = """
|
322
316
|
|ignore_on_server:
|
323
317
|
| - .*txt
|
@@ -329,14 +323,14 @@ class S3WebsiteSpec extends Specification {
|
|
329
323
|
}
|
330
324
|
|
331
325
|
"max-age in config" can {
|
332
|
-
"be applied to all files" in new EmptySite with VerboseLogger with MockAWS {
|
326
|
+
"be applied to all files" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
333
327
|
config = "max_age: 60"
|
334
328
|
setLocalFile("index.html")
|
335
329
|
Push.pushSite
|
336
330
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
337
331
|
}
|
338
332
|
|
339
|
-
"be applied to files that match the glob" in new EmptySite with VerboseLogger with MockAWS {
|
333
|
+
"be applied to files that match the glob" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
340
334
|
config = """
|
341
335
|
|max_age:
|
342
336
|
| "*.html": 90
|
@@ -346,7 +340,7 @@ class S3WebsiteSpec extends Specification {
|
|
346
340
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
347
341
|
}
|
348
342
|
|
349
|
-
"be applied to directories that match the glob" in new EmptySite with VerboseLogger with MockAWS {
|
343
|
+
"be applied to directories that match the glob" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
350
344
|
config = """
|
351
345
|
|max_age:
|
352
346
|
| "assets/**/*.js": 90
|
@@ -356,7 +350,7 @@ class S3WebsiteSpec extends Specification {
|
|
356
350
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
357
351
|
}
|
358
352
|
|
359
|
-
"not be applied if the glob doesn't match" in new EmptySite with VerboseLogger with MockAWS {
|
353
|
+
"not be applied if the glob doesn't match" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
360
354
|
config = """
|
361
355
|
|max_age:
|
362
356
|
| "*.js": 90
|
@@ -366,7 +360,7 @@ class S3WebsiteSpec extends Specification {
|
|
366
360
|
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
367
361
|
}
|
368
362
|
|
369
|
-
"be used to disable caching" in new EmptySite with VerboseLogger with MockAWS {
|
363
|
+
"be used to disable caching" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
370
364
|
config = "max_age: 0"
|
371
365
|
setLocalFile("index.html")
|
372
366
|
Push.pushSite
|
@@ -375,7 +369,7 @@ class S3WebsiteSpec extends Specification {
|
|
375
369
|
}
|
376
370
|
|
377
371
|
"max-age in config" should {
|
378
|
-
"respect the more specific glob" in new EmptySite with VerboseLogger with MockAWS {
|
372
|
+
"respect the more specific glob" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
379
373
|
config = """
|
380
374
|
|max_age:
|
381
375
|
| "assets/*": 150
|
@@ -389,7 +383,7 @@ class S3WebsiteSpec extends Specification {
|
|
389
383
|
}
|
390
384
|
|
391
385
|
"s3_reduced_redundancy: true in config" should {
|
392
|
-
"result in uploads being marked with reduced redundancy" in new EmptySite with VerboseLogger with MockAWS {
|
386
|
+
"result in uploads being marked with reduced redundancy" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
393
387
|
config = "s3_reduced_redundancy: true"
|
394
388
|
setLocalFile("file.exe")
|
395
389
|
Push.pushSite
|
@@ -398,7 +392,7 @@ class S3WebsiteSpec extends Specification {
|
|
398
392
|
}
|
399
393
|
|
400
394
|
"s3_reduced_redundancy: false in config" should {
|
401
|
-
"result in uploads being marked with the default storage class" in new EmptySite with VerboseLogger with MockAWS {
|
395
|
+
"result in uploads being marked with the default storage class" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
402
396
|
config = "s3_reduced_redundancy: false"
|
403
397
|
setLocalFile("file.exe")
|
404
398
|
Push.pushSite
|
@@ -407,7 +401,7 @@ class S3WebsiteSpec extends Specification {
|
|
407
401
|
}
|
408
402
|
|
409
403
|
"redirect in config" should {
|
410
|
-
"result in a redirect instruction that is sent to AWS" in new EmptySite with VerboseLogger with MockAWS {
|
404
|
+
"result in a redirect instruction that is sent to AWS" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
411
405
|
config = """
|
412
406
|
|redirects:
|
413
407
|
| index.php: /index.html
|
@@ -416,7 +410,7 @@ class S3WebsiteSpec extends Specification {
|
|
416
410
|
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
417
411
|
}
|
418
412
|
|
419
|
-
"result in max-age=0 Cache-Control header on the object" in new EmptySite with VerboseLogger with MockAWS {
|
413
|
+
"result in max-age=0 Cache-Control header on the object" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
420
414
|
config = """
|
421
415
|
|redirects:
|
422
416
|
| index.php: /index.html
|
@@ -427,7 +421,7 @@ class S3WebsiteSpec extends Specification {
|
|
427
421
|
}
|
428
422
|
|
429
423
|
"redirect in config and an object on the S3 bucket" should {
|
430
|
-
"not result in the S3 object being deleted" in new EmptySite with VerboseLogger with MockAWS {
|
424
|
+
"not result in the S3 object being deleted" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
431
425
|
config = """
|
432
426
|
|redirects:
|
433
427
|
| index.php: /index.html
|
@@ -440,7 +434,7 @@ class S3WebsiteSpec extends Specification {
|
|
440
434
|
}
|
441
435
|
|
442
436
|
"dotfiles" should {
|
443
|
-
"be included in the pushed files" in new EmptySite with VerboseLogger with MockAWS {
|
437
|
+
"be included in the pushed files" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
444
438
|
setLocalFile(".vimrc")
|
445
439
|
Push.pushSite
|
446
440
|
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
@@ -448,25 +442,25 @@ class S3WebsiteSpec extends Specification {
|
|
448
442
|
}
|
449
443
|
|
450
444
|
"content type inference" should {
|
451
|
-
"add charset=utf-8 to all html documents" in new EmptySite with VerboseLogger with MockAWS {
|
445
|
+
"add charset=utf-8 to all html documents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
452
446
|
setLocalFile("index.html")
|
453
447
|
Push.pushSite
|
454
448
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
455
449
|
}
|
456
450
|
|
457
|
-
"add charset=utf-8 to all text documents" in new EmptySite with VerboseLogger with MockAWS {
|
451
|
+
"add charset=utf-8 to all text documents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
458
452
|
setLocalFile("index.txt")
|
459
453
|
Push.pushSite
|
460
454
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/plain; charset=utf-8")
|
461
455
|
}
|
462
456
|
|
463
|
-
"add charset=utf-8 to all json documents" in new EmptySite with VerboseLogger with MockAWS {
|
457
|
+
"add charset=utf-8 to all json documents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
464
458
|
setLocalFile("data.json")
|
465
459
|
Push.pushSite
|
466
460
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("application/json; charset=utf-8")
|
467
461
|
}
|
468
462
|
|
469
|
-
"resolve the content type from file contents" in new EmptySite with VerboseLogger with MockAWS {
|
463
|
+
"resolve the content type from file contents" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
470
464
|
setLocalFileWithContent(("index", "<html><body><h1>hi</h1></body></html>"))
|
471
465
|
Push.pushSite
|
472
466
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
@@ -474,7 +468,7 @@ class S3WebsiteSpec extends Specification {
|
|
474
468
|
}
|
475
469
|
|
476
470
|
"ERB in config file" should {
|
477
|
-
"be evaluated" in new EmptySite with VerboseLogger with MockAWS {
|
471
|
+
"be evaluated" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
478
472
|
config = """
|
479
473
|
|redirects:
|
480
474
|
|<%= ('a'..'f').to_a.map do |t| ' '+t+ ': /'+t+'.html' end.join('\n')%>
|
@@ -486,17 +480,67 @@ class S3WebsiteSpec extends Specification {
|
|
486
480
|
}
|
487
481
|
|
488
482
|
"logging" should {
|
489
|
-
"print the debug messages when --verbose is defined" in new EmptySite with VerboseLogger with MockAWS {
|
483
|
+
"print the debug messages when --verbose is defined" in new EmptySite with VerboseLogger with MockAWS with DefaultRunMode {
|
490
484
|
Push.pushSite
|
491
485
|
logEntries must contain("[debg] Querying S3 files")
|
492
486
|
}
|
493
487
|
|
494
|
-
"not print the debug messages by default" in new EmptySite with NonVerboseLogger with MockAWS {
|
488
|
+
"not print the debug messages by default" in new EmptySite with NonVerboseLogger with MockAWS with DefaultRunMode {
|
495
489
|
Push.pushSite
|
496
490
|
logEntries.forall(_.contains("[debg]")) must beFalse
|
497
491
|
}
|
498
492
|
}
|
499
493
|
|
494
|
+
"dry run" should {
|
495
|
+
"not push updates" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
496
|
+
setLocalFileWithContent(("index.html", "<div>new</div>"))
|
497
|
+
setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
|
498
|
+
Push.pushSite
|
499
|
+
noUploadsOccurred must beTrue
|
500
|
+
}
|
501
|
+
|
502
|
+
"not push redirects" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
503
|
+
config =
|
504
|
+
"""
|
505
|
+
|redirects:
|
506
|
+
| index.php: /index.html
|
507
|
+
""".stripMargin
|
508
|
+
Push.pushSite
|
509
|
+
noUploadsOccurred must beTrue
|
510
|
+
}
|
511
|
+
|
512
|
+
"not push deletes" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
513
|
+
setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
|
514
|
+
Push.pushSite
|
515
|
+
noUploadsOccurred must beTrue
|
516
|
+
}
|
517
|
+
|
518
|
+
"not push new files" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
519
|
+
setLocalFile("index.html")
|
520
|
+
Push.pushSite
|
521
|
+
noUploadsOccurred must beTrue
|
522
|
+
}
|
523
|
+
|
524
|
+
"not invalidate files" in new EmptySite with VerboseLogger with MockAWS with DryRunMode {
|
525
|
+
config = "cloudfront_invalidation_id: AABBCC"
|
526
|
+
setS3Files(S3File("index.html", md5Hex("<div>old</div>")))
|
527
|
+
Push.pushSite
|
528
|
+
noInvalidationsOccurred must beTrue
|
529
|
+
}
|
530
|
+
}
|
531
|
+
|
532
|
+
trait DefaultRunMode {
|
533
|
+
implicit def pushMode: PushMode = new PushMode {
|
534
|
+
def dryRun = false
|
535
|
+
}
|
536
|
+
}
|
537
|
+
|
538
|
+
trait DryRunMode {
|
539
|
+
implicit def pushMode: PushMode = new PushMode {
|
540
|
+
def dryRun = true
|
541
|
+
}
|
542
|
+
}
|
543
|
+
|
500
544
|
trait MockAWS extends MockS3 with MockCloudFront with Scope
|
501
545
|
|
502
546
|
trait VerboseLogger extends LogCapturer {
|
@@ -518,7 +562,7 @@ class S3WebsiteSpec extends Specification {
|
|
518
562
|
|
519
563
|
trait MockCloudFront extends MockAWSHelper {
|
520
564
|
val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
|
521
|
-
implicit val cfSettings:
|
565
|
+
implicit val cfSettings: CloudFrontSetting = CloudFrontSetting(
|
522
566
|
cfClient = _ => amazonCloudFrontClient,
|
523
567
|
retryTimeUnit = MICROSECONDS
|
524
568
|
)
|
@@ -564,7 +608,7 @@ class S3WebsiteSpec extends Specification {
|
|
564
608
|
|
565
609
|
trait MockS3 extends MockAWSHelper {
|
566
610
|
val amazonS3Client = mock(classOf[AmazonS3])
|
567
|
-
implicit val s3Settings:
|
611
|
+
implicit val s3Settings: S3Setting = S3Setting(
|
568
612
|
s3Client = _ => amazonS3Client,
|
569
613
|
retryTimeUnit = MICROSECONDS
|
570
614
|
)
|
@@ -590,8 +634,6 @@ class S3WebsiteSpec extends Specification {
|
|
590
634
|
}
|
591
635
|
}
|
592
636
|
|
593
|
-
val s3 = new S3()
|
594
|
-
|
595
637
|
def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
596
638
|
doAnswer(temporaryFailure(classOf[PutObjectResult]))
|
597
639
|
.when(amazonS3Client)
|
@@ -611,7 +653,6 @@ class S3WebsiteSpec extends Specification {
|
|
611
653
|
}
|
612
654
|
|
613
655
|
def asSeenByS3Client(upload: Upload)(implicit config: Config, logger: Logger): PutObjectRequest = {
|
614
|
-
Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
|
615
656
|
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
616
657
|
verify(amazonS3Client).putObject(req.capture())
|
617
658
|
req.getValue
|