s3_website_monadic 0.0.26 → 0.0.27
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/s3_website_monadic +40 -23
- data/lib/s3_website/version.rb +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +4 -5
- data/src/main/scala/s3/website/Push.scala +15 -14
- data/src/main/scala/s3/website/S3.scala +8 -7
- data/src/main/scala/s3/website/Utils.scala +6 -6
- data/src/main/scala/s3/website/model/Site.scala +5 -4
- data/src/main/scala/s3/website/package.scala +6 -5
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +83 -53
- metadata +2 -3
- data/src/main/scala/s3/website/Implicits.scala +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f19d0eae1ea4ae3a515bc6f26066a2a90b241d58
|
4
|
+
data.tar.gz: 3c8762e1cda26b3f84021b3a071c75c61f1f6e87
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0eb150f681d4cb2efd22a35a26ee55df9fe4575e560b82a17a0043161deea72a2979839679aec20d3bfefe400a2a46389a93378f346b3da7d55597b34a44d394
|
7
|
+
data.tar.gz: 5937ab122bf4014de4073c1796c4f43cadd38488e2d893d8715cbfeebd3acb6e3a0808976bbd630aeaa03609c0f58e0a8426ad9f8ccc9f58208139e75b1b9b7e
|
data/bin/s3_website_monadic
CHANGED
@@ -53,6 +53,12 @@ class Cli < Thor
|
|
53
53
|
:default => Dir.pwd,
|
54
54
|
:desc => "The directory where your config file is. When not defined, s3_website will look in the current working directory."
|
55
55
|
)
|
56
|
+
option(
|
57
|
+
:verbose,
|
58
|
+
:type => :boolean,
|
59
|
+
:default => false,
|
60
|
+
:desc => "Print verbose output"
|
61
|
+
)
|
56
62
|
desc 'push', 'Push local files with the S3 website'
|
57
63
|
long_desc <<-LONGDESC
|
58
64
|
`s3_website push` will upload new and changes files to S3. It will
|
@@ -61,10 +67,11 @@ class Cli < Thor
|
|
61
67
|
def push
|
62
68
|
call_dir = Dir.pwd
|
63
69
|
project_root = File.expand_path(File.dirname(__FILE__)+ '/..')
|
70
|
+
logger = Logger.new(options[:verbose])
|
64
71
|
Dir.chdir(project_root) {
|
65
|
-
jar_file = resolve_jar project_root
|
72
|
+
jar_file = resolve_jar project_root, logger
|
66
73
|
# Then run it
|
67
|
-
run_s3_website_jar(jar_file, call_dir)
|
74
|
+
run_s3_website_jar(jar_file, call_dir, logger)
|
68
75
|
}
|
69
76
|
end
|
70
77
|
|
@@ -72,11 +79,11 @@ class Cli < Thor
|
|
72
79
|
subcommand 'cfg', Cfg
|
73
80
|
end
|
74
81
|
|
75
|
-
def run_s3_website_jar(jar_file, call_dir)
|
82
|
+
def run_s3_website_jar(jar_file, call_dir, logger)
|
76
83
|
site_path = File.expand_path(S3Website::Paths.infer_site_path options[:site], call_dir)
|
77
84
|
config_dir = File.expand_path options[:config_dir]
|
78
|
-
args = "--site=#{site_path} --config-dir=#{config_dir}"
|
79
|
-
debug_msg "Using #{jar_file}"
|
85
|
+
args = "--site=#{site_path} --config-dir=#{config_dir} #{'--verbose' if options[:verbose]}"
|
86
|
+
logger.debug_msg "Using #{jar_file}"
|
80
87
|
if system("java -cp #{jar_file} s3.website.Push #{args}")
|
81
88
|
exit 0
|
82
89
|
else
|
@@ -84,7 +91,7 @@ def run_s3_website_jar(jar_file, call_dir)
|
|
84
91
|
end
|
85
92
|
end
|
86
93
|
|
87
|
-
def resolve_jar(project_root)
|
94
|
+
def resolve_jar(project_root, logger)
|
88
95
|
development_jar_path =
|
89
96
|
project_root + '/target/scala-2.11/s3_website.jar'
|
90
97
|
released_jar_lookup_paths = [
|
@@ -103,26 +110,25 @@ def resolve_jar(project_root)
|
|
103
110
|
is_development = File.exists?(project_root + '/.git')
|
104
111
|
if is_development
|
105
112
|
system "./sbt assembly"
|
106
|
-
puts development_jar_path
|
107
113
|
development_jar_path
|
108
114
|
else
|
109
|
-
download_jar(released_jar_lookup_paths)
|
115
|
+
download_jar(released_jar_lookup_paths, logger)
|
110
116
|
end
|
111
117
|
end
|
112
118
|
end
|
113
119
|
|
114
|
-
def download_jar(released_jar_lookup_paths)
|
120
|
+
def download_jar(released_jar_lookup_paths, logger)
|
115
121
|
tag_name = "v#{S3Website::VERSION}"
|
116
122
|
downloaded_jar = released_jar_lookup_paths.select { |jar_path|
|
117
123
|
File.writable? File.dirname(jar_path)
|
118
124
|
}.first
|
119
125
|
unless downloaded_jar
|
120
|
-
fail_msg "Neither #{released_jar_lookup_paths.join ' or '} is writable. Cannot download s3_website.jar."
|
121
|
-
fail_msg "Set either directory as writable to the current user and try again."
|
126
|
+
logger.fail_msg "Neither #{released_jar_lookup_paths.join ' or '} is writable. Cannot download s3_website.jar."
|
127
|
+
logger.fail_msg "Set either directory as writable to the current user and try again."
|
122
128
|
exit 1
|
123
129
|
end
|
124
130
|
download_url = "https://github.com/laurilehmijoki/s3_website/releases/download/#{tag_name}/s3_website.jar"
|
125
|
-
info_msg "Downloading #{download_url} into #{downloaded_jar}"
|
131
|
+
logger.info_msg "Downloading #{download_url} into #{downloaded_jar}"
|
126
132
|
require 'open-uri'
|
127
133
|
open(downloaded_jar, 'wb') do |file|
|
128
134
|
file << open(download_url).read
|
@@ -130,20 +136,31 @@ def download_jar(released_jar_lookup_paths)
|
|
130
136
|
downloaded_jar
|
131
137
|
end
|
132
138
|
|
133
|
-
|
134
|
-
|
135
|
-
|
139
|
+
class Logger
|
140
|
+
attr_reader :verbose
|
141
|
+
def initialize(verbose)
|
142
|
+
@verbose = verbose
|
143
|
+
end
|
136
144
|
|
137
|
-
def
|
138
|
-
|
139
|
-
|
145
|
+
def debug_msg(msg)
|
146
|
+
if verbose
|
147
|
+
print_msg 'debg'.cyan, msg
|
148
|
+
end
|
149
|
+
end
|
140
150
|
|
141
|
-
def
|
142
|
-
|
143
|
-
end
|
151
|
+
def info_msg(msg)
|
152
|
+
print_msg 'info'.blue, msg
|
153
|
+
end
|
154
|
+
|
155
|
+
def fail_msg(msg)
|
156
|
+
print_msg 'fail'.red, msg
|
157
|
+
end
|
158
|
+
|
159
|
+
private
|
144
160
|
|
145
|
-
def print_msg(prefix, msg)
|
146
|
-
|
161
|
+
def print_msg(prefix, msg)
|
162
|
+
puts "[#{prefix}] #{msg}"
|
163
|
+
end
|
147
164
|
end
|
148
165
|
|
149
166
|
Cli.start(ARGV)
|
data/lib/s3_website/version.rb
CHANGED
@@ -8,12 +8,11 @@ import scala.collection.JavaConversions._
|
|
8
8
|
import scala.concurrent.duration._
|
9
9
|
import s3.website.S3.{SuccessfulDelete, PushSuccessReport, SuccessfulUpload}
|
10
10
|
import com.amazonaws.auth.BasicAWSCredentials
|
11
|
-
import s3.website.Logger._
|
12
11
|
import java.net.URI
|
13
12
|
import Utils._
|
14
13
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
15
14
|
|
16
|
-
class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config) {
|
15
|
+
class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config, logger: Logger) {
|
17
16
|
val cloudFront = cloudFrontSettings.cfClient(config)
|
18
17
|
|
19
18
|
def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
|
@@ -22,7 +21,7 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config
|
|
22
21
|
val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
|
23
22
|
cloudFront.createInvalidation(invalidationReq)
|
24
23
|
val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
|
25
|
-
info(result)
|
24
|
+
logger.info(result)
|
26
25
|
Right(result)
|
27
26
|
} recoverWith (tooManyInvalidationsRetry(invalidationBatch, distributionId, attempt) orElse retry(attempt)(
|
28
27
|
createFailureReport = error => FailedInvalidation(error),
|
@@ -30,13 +29,13 @@ class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config
|
|
30
29
|
))
|
31
30
|
|
32
31
|
def tooManyInvalidationsRetry(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt)
|
33
|
-
(implicit ec: ExecutionContextExecutor): PartialFunction[Throwable, InvalidationResult] = {
|
32
|
+
(implicit ec: ExecutionContextExecutor, logger: Logger): PartialFunction[Throwable, InvalidationResult] = {
|
34
33
|
case e: TooManyInvalidationsInProgressException =>
|
35
34
|
val duration: Duration = Duration(
|
36
35
|
(fibs drop attempt).head min 15, /* CloudFront invalidations complete within 15 minutes */
|
37
36
|
cloudFrontSettings.retryTimeUnit
|
38
37
|
)
|
39
|
-
pending(maxInvalidationsExceededInfo(duration, attempt))
|
38
|
+
logger.pending(maxInvalidationsExceededInfo(duration, attempt))
|
40
39
|
Thread.sleep(duration.toMillis)
|
41
40
|
invalidate(invalidationBatch, distributionId, attempt + 1)
|
42
41
|
}
|
@@ -13,13 +13,11 @@ import s3.website.model.LocalFile.resolveLocalFiles
|
|
13
13
|
import scala.collection.parallel.ParSeq
|
14
14
|
import java.util.concurrent.ExecutorService
|
15
15
|
import s3.website.model._
|
16
|
-
import s3.website.Implicits._
|
17
16
|
import s3.website.model.Update
|
18
17
|
import s3.website.model.NewFile
|
19
18
|
import s3.website.S3.PushSuccessReport
|
20
19
|
import scala.collection.mutable.ArrayBuffer
|
21
20
|
import s3.website.CloudFront._
|
22
|
-
import s3.website.Logger._
|
23
21
|
import s3.website.S3.SuccessfulDelete
|
24
22
|
import s3.website.CloudFront.SuccessfulInvalidation
|
25
23
|
import s3.website.S3.S3Settings
|
@@ -33,10 +31,11 @@ object Push {
|
|
33
31
|
implicit site: Site,
|
34
32
|
executor: ExecutionContextExecutor,
|
35
33
|
s3Settings: S3Settings,
|
36
|
-
cloudFrontSettings: CloudFrontSettings
|
34
|
+
cloudFrontSettings: CloudFrontSettings,
|
35
|
+
logger: Logger
|
37
36
|
): ExitCode = {
|
38
|
-
info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
39
|
-
val utils
|
37
|
+
logger.info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
38
|
+
val utils = new Utils
|
40
39
|
|
41
40
|
val redirects = Redirect.resolveRedirects
|
42
41
|
val redirectResults = redirects.map(new S3() upload(_))
|
@@ -64,7 +63,7 @@ object Push {
|
|
64
63
|
|
65
64
|
def invalidateCloudFrontItems
|
66
65
|
(errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
|
67
|
-
(implicit config: Config, cloudFrontSettings: CloudFrontSettings, ec: ExecutionContextExecutor): Option[InvalidationSucceeded] = {
|
66
|
+
(implicit config: Config, cloudFrontSettings: CloudFrontSettings, ec: ExecutionContextExecutor, logger: Logger): Option[InvalidationSucceeded] = {
|
68
67
|
config.cloudfront_distribution_id.map {
|
69
68
|
distributionId =>
|
70
69
|
val pushSuccessReports = errorsOrFinishedPushOps.fold(
|
@@ -101,12 +100,13 @@ object Push {
|
|
101
100
|
|
102
101
|
type InvalidationSucceeded = Boolean
|
103
102
|
|
104
|
-
def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])
|
103
|
+
def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])
|
104
|
+
(implicit config: Config, logger: Logger): ExitCode = {
|
105
105
|
errorsOrFinishedUploads.right.foreach { finishedUploads =>
|
106
106
|
val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
|
107
|
-
info(s"Summary: $pushCounts")
|
107
|
+
logger.info(s"Summary: $pushCounts")
|
108
108
|
}
|
109
|
-
errorsOrFinishedUploads.left foreach (err => fail(s"Encountered an error: ${err.reportMessage}"))
|
109
|
+
errorsOrFinishedUploads.left foreach (err => logger.fail(s"Encountered an error: ${err.reportMessage}"))
|
110
110
|
val exitCode = errorsOrFinishedUploads.fold(
|
111
111
|
_ => 1,
|
112
112
|
finishedUploads => finishedUploads.foldLeft(0) { (memo, finishedUpload) =>
|
@@ -121,9 +121,9 @@ object Push {
|
|
121
121
|
)
|
122
122
|
|
123
123
|
if (exitCode == 0)
|
124
|
-
info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
124
|
+
logger.info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
125
125
|
else
|
126
|
-
fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
126
|
+
logger.fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
127
127
|
exitCode
|
128
128
|
}
|
129
129
|
|
@@ -181,20 +181,21 @@ object Push {
|
|
181
181
|
|
182
182
|
@Option def site: String
|
183
183
|
@Option(longName = Array("config-dir")) def configDir: String
|
184
|
-
|
184
|
+
@Option def verbose: Boolean
|
185
185
|
}
|
186
186
|
|
187
187
|
def main(args: Array[String]) {
|
188
188
|
val cliArgs = CliFactory.parseArguments(classOf[CliArgs], args:_*)
|
189
189
|
implicit val s3Settings = S3Settings()
|
190
190
|
implicit val cloudFrontSettings = CloudFrontSettings()
|
191
|
+
implicit val logger: Logger = new Logger(cliArgs.verbose)
|
191
192
|
val errorOrPushStatus = push(siteInDirectory = cliArgs.site, withConfigDirectory = cliArgs.configDir)
|
192
|
-
errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.reportMessage}"))
|
193
|
+
errorOrPushStatus.left foreach (err => logger.fail(s"Could not load the site: ${err.reportMessage}"))
|
193
194
|
System exit errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus)
|
194
195
|
}
|
195
196
|
|
196
197
|
def push(siteInDirectory: String, withConfigDirectory: String)
|
197
|
-
(implicit s3Settings: S3Settings, cloudFrontSettings: CloudFrontSettings) =
|
198
|
+
(implicit s3Settings: S3Settings, cloudFrontSettings: CloudFrontSettings, logger: Logger) =
|
198
199
|
loadSite(withConfigDirectory + "/s3_website.yml", siteInDirectory)
|
199
200
|
.right
|
200
201
|
.map {
|
@@ -7,7 +7,6 @@ import com.amazonaws.services.s3.model._
|
|
7
7
|
import scala.collection.JavaConversions._
|
8
8
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
9
9
|
import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
|
10
|
-
import s3.website.Logger._
|
11
10
|
import scala.concurrent.duration.TimeUnit
|
12
11
|
import java.util.concurrent.TimeUnit.SECONDS
|
13
12
|
import s3.website.S3.SuccessfulUpload
|
@@ -19,23 +18,25 @@ import s3.website.S3.S3Settings
|
|
19
18
|
|
20
19
|
class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
|
21
20
|
|
22
|
-
def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)
|
21
|
+
def upload(upload: Upload with UploadTypeResolved, a: Attempt = 1)
|
22
|
+
(implicit config: Config, logger: Logger): Future[Either[FailedUpload, SuccessfulUpload]] =
|
23
23
|
Future {
|
24
24
|
val putObjectRequest = toPutObjectRequest(upload)
|
25
25
|
s3Settings.s3Client(config) putObject putObjectRequest
|
26
26
|
val report = SuccessfulUpload(upload, putObjectRequest)
|
27
|
-
info(report)
|
27
|
+
logger.info(report)
|
28
28
|
Right(report)
|
29
29
|
} recoverWith retry(a)(
|
30
30
|
createFailureReport = error => FailedUpload(upload.s3Key, error),
|
31
31
|
retryAction = newAttempt => this.upload(upload, newAttempt)
|
32
32
|
)
|
33
33
|
|
34
|
-
def delete(s3Key: String, a: Attempt = 1)
|
34
|
+
def delete(s3Key: String, a: Attempt = 1)
|
35
|
+
(implicit config: Config, logger: Logger): Future[Either[FailedDelete, SuccessfulDelete]] =
|
35
36
|
Future {
|
36
37
|
s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
37
38
|
val report = SuccessfulDelete(s3Key)
|
38
|
-
info(report)
|
39
|
+
logger.info(report)
|
39
40
|
Right(report)
|
40
41
|
} recoverWith retry(a)(
|
41
42
|
createFailureReport = error => FailedDelete(s3Key, error),
|
@@ -83,9 +84,9 @@ object S3 {
|
|
83
84
|
|
84
85
|
def resolveS3FilesAndUpdates(localFiles: Seq[LocalFile])
|
85
86
|
(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil, attempt: Attempt = 1, onFlightUpdateFutures: UpdateFutures = Nil)
|
86
|
-
(implicit config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor):
|
87
|
+
(implicit config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor, logger: Logger):
|
87
88
|
ErrorOrS3FilesAndUpdates = Future {
|
88
|
-
debug(nextMarker.fold
|
89
|
+
logger.debug(nextMarker.fold
|
89
90
|
("Querying S3 files")
|
90
91
|
{m => s"Querying more S3 files (starting from $m)"}
|
91
92
|
)
|
@@ -16,16 +16,16 @@ object Utils {
|
|
16
16
|
lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
|
17
17
|
}
|
18
18
|
|
19
|
-
|
19
|
+
class Logger(verboseOutput: Boolean, logMessage: (String) => Unit = println) {
|
20
20
|
import Rainbow._
|
21
|
-
def debug(msg: String) =
|
22
|
-
def info(msg: String) =
|
23
|
-
def fail(msg: String) =
|
21
|
+
def debug(msg: String) = if (verboseOutput) logMessage(s"[${"debg".cyan}] $msg")
|
22
|
+
def info(msg: String) = logMessage(s"[${"info".blue}] $msg")
|
23
|
+
def fail(msg: String) = logMessage(s"[${"fail".red}] $msg")
|
24
24
|
|
25
|
-
def info(report: SuccessReport) =
|
25
|
+
def info(report: SuccessReport) = logMessage(s"[${"succ".green}] ${report.reportMessage}")
|
26
26
|
def info(report: FailureReport) = fail(report.reportMessage)
|
27
27
|
|
28
|
-
def pending(msg: String) =
|
28
|
+
def pending(msg: String) = logMessage(s"[${"wait".yellow}] $msg")
|
29
29
|
}
|
30
30
|
|
31
31
|
/**
|
@@ -6,7 +6,7 @@ import org.yaml.snakeyaml.Yaml
|
|
6
6
|
import s3.website.model.Config._
|
7
7
|
import scala.io.Source.fromFile
|
8
8
|
import scala.language.postfixOps
|
9
|
-
import s3.website.Logger
|
9
|
+
import s3.website.Logger
|
10
10
|
import scala.util.Failure
|
11
11
|
import s3.website.model.Config.UnsafeYaml
|
12
12
|
import scala.util.Success
|
@@ -17,7 +17,8 @@ case class Site(rootDirectory: String, config: Config) {
|
|
17
17
|
}
|
18
18
|
|
19
19
|
object Site {
|
20
|
-
def loadSite(yamlConfigPath: String, siteRootDirectory: String)
|
20
|
+
def loadSite(yamlConfigPath: String, siteRootDirectory: String)
|
21
|
+
(implicit logger: Logger): Either[ErrorReport, Site] = {
|
21
22
|
val yamlObjectTry = for {
|
22
23
|
yamlString <- Try(fromFile(new File(yamlConfigPath)).mkString)
|
23
24
|
yamlWithErbEvaluated <- erbEval(yamlString, yamlConfigPath)
|
@@ -43,8 +44,8 @@ object Site {
|
|
43
44
|
concurrency_level <- loadOptionalInt("concurrency_level").right
|
44
45
|
redirects <- loadRedirects.right
|
45
46
|
} yield {
|
46
|
-
gzip_zopfli.foreach(_ => info("zopfli is not currently supported"))
|
47
|
-
extensionless_mime_type.foreach(_ => info(
|
47
|
+
gzip_zopfli.foreach(_ => logger.info("zopfli is not currently supported"))
|
48
|
+
extensionless_mime_type.foreach(_ => logger.info(
|
48
49
|
s"Ignoring the extensionless_mime_type setting in $yamlConfigPath. Counting on Apache Tika to infer correct mime types.")
|
49
50
|
)
|
50
51
|
Config(
|
@@ -1,12 +1,11 @@
|
|
1
1
|
package s3
|
2
2
|
|
3
3
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
4
|
-
import s3.website.Logger._
|
5
4
|
import scala.concurrent.duration.{TimeUnit, Duration}
|
6
5
|
import s3.website.Utils._
|
7
6
|
import s3.website.S3.{PushSuccessReport, PushFailureReport}
|
8
|
-
import com.amazonaws.services.s3.model.AmazonS3Exception
|
9
7
|
import com.amazonaws.AmazonServiceException
|
8
|
+
import s3.website.model.{Config, Site}
|
10
9
|
|
11
10
|
package object website {
|
12
11
|
trait Report {
|
@@ -28,16 +27,16 @@ package object website {
|
|
28
27
|
|
29
28
|
def retry[L <: Report, R](attempt: Attempt)
|
30
29
|
(createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
|
31
|
-
(implicit retrySettings: RetrySettings, ec: ExecutionContextExecutor):
|
30
|
+
(implicit retrySettings: RetrySettings, ec: ExecutionContextExecutor, logger: Logger):
|
32
31
|
PartialFunction[Throwable, Future[Either[L, R]]] = {
|
33
32
|
case error: Throwable if attempt == 6 || isIrrecoverable(error) =>
|
34
33
|
val failureReport = createFailureReport(error)
|
35
|
-
fail(failureReport.reportMessage)
|
34
|
+
logger.fail(failureReport.reportMessage)
|
36
35
|
Future(Left(failureReport))
|
37
36
|
case error: Throwable =>
|
38
37
|
val failureReport = createFailureReport(error)
|
39
38
|
val sleepDuration = Duration(fibs.drop(attempt + 1).head, retrySettings.retryTimeUnit)
|
40
|
-
pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
39
|
+
logger.pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
41
40
|
Thread.sleep(sleepDuration.toMillis)
|
42
41
|
retryAction(attempt + 1)
|
43
42
|
}
|
@@ -59,4 +58,6 @@ package object website {
|
|
59
58
|
s"$count ${if (count > 1) plural else singular}"
|
60
59
|
}
|
61
60
|
}
|
61
|
+
|
62
|
+
implicit def site2Config(implicit site: Site): Config = site.config
|
62
63
|
}
|
@@ -30,7 +30,7 @@ import scala.collection.mutable
|
|
30
30
|
class S3WebsiteSpec extends Specification {
|
31
31
|
|
32
32
|
"gzip: true" should {
|
33
|
-
"update a gzipped S3 object if the contents has changed" in new EmptySite with MockAWS {
|
33
|
+
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS {
|
34
34
|
config = "gzip: true"
|
35
35
|
setLocalFileWithContent(("styles.css", "<h1>hi again</h1>"))
|
36
36
|
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
@@ -38,7 +38,7 @@ class S3WebsiteSpec extends Specification {
|
|
38
38
|
sentPutObjectRequest.getKey must equalTo("styles.css")
|
39
39
|
}
|
40
40
|
|
41
|
-
"not update a gzipped S3 object if the contents has not changed" in new EmptySite with MockAWS {
|
41
|
+
"not update a gzipped S3 object if the contents has not changed" in new EmptySite with VerboseLogger with MockAWS {
|
42
42
|
config = "gzip: true"
|
43
43
|
setLocalFileWithContent(("styles.css", "<h1>hi</h1>"))
|
44
44
|
setS3Files(S3File("styles.css", "1c5117e5839ad8fc00ce3c41296255a1" /* md5 of the gzip of the file contents */))
|
@@ -51,7 +51,7 @@ class S3WebsiteSpec extends Specification {
|
|
51
51
|
gzip:
|
52
52
|
- .xml
|
53
53
|
""" should {
|
54
|
-
"update a gzipped S3 object if the contents has changed" in new EmptySite with MockAWS {
|
54
|
+
"update a gzipped S3 object if the contents has changed" in new EmptySite with VerboseLogger with MockAWS {
|
55
55
|
config = """
|
56
56
|
|gzip:
|
57
57
|
| - .xml
|
@@ -64,40 +64,40 @@ class S3WebsiteSpec extends Specification {
|
|
64
64
|
}
|
65
65
|
|
66
66
|
"push" should {
|
67
|
-
"not upload a file if it has not changed" in new EmptySite with MockAWS {
|
67
|
+
"not upload a file if it has not changed" in new EmptySite with VerboseLogger with MockAWS {
|
68
68
|
setLocalFileWithContent(("index.html", "<div>hello</div>"))
|
69
69
|
setS3Files(S3File("index.html", md5Hex("<div>hello</div>")))
|
70
70
|
Push.pushSite
|
71
71
|
noUploadsOccurred must beTrue
|
72
72
|
}
|
73
73
|
|
74
|
-
"update a file if it has changed" in new EmptySite with MockAWS {
|
74
|
+
"update a file if it has changed" in new EmptySite with VerboseLogger with MockAWS {
|
75
75
|
setLocalFileWithContent(("index.html", "<h1>old text</h1>"))
|
76
76
|
setS3Files(S3File("index.html", md5Hex("<h1>new text</h1>")))
|
77
77
|
Push.pushSite
|
78
78
|
sentPutObjectRequest.getKey must equalTo("index.html")
|
79
79
|
}
|
80
80
|
|
81
|
-
"create a file if does not exist on S3" in new EmptySite with MockAWS {
|
81
|
+
"create a file if does not exist on S3" in new EmptySite with VerboseLogger with MockAWS {
|
82
82
|
setLocalFile("index.html")
|
83
83
|
Push.pushSite
|
84
84
|
sentPutObjectRequest.getKey must equalTo("index.html")
|
85
85
|
}
|
86
86
|
|
87
|
-
"delete files that are on S3 but not on local file system" in new EmptySite with MockAWS {
|
87
|
+
"delete files that are on S3 but not on local file system" in new EmptySite with VerboseLogger with MockAWS {
|
88
88
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
89
89
|
Push.pushSite
|
90
90
|
sentDelete must equalTo("old.html")
|
91
91
|
}
|
92
92
|
|
93
|
-
"try again if the upload fails" in new EmptySite with MockAWS {
|
93
|
+
"try again if the upload fails" in new EmptySite with VerboseLogger with MockAWS {
|
94
94
|
setLocalFile("index.html")
|
95
95
|
uploadFailsAndThenSucceeds(howManyFailures = 5)
|
96
96
|
Push.pushSite
|
97
97
|
verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
98
98
|
}
|
99
99
|
|
100
|
-
"not try again if the upload fails on because of invalid credentials" in new EmptySite with MockAWS {
|
100
|
+
"not try again if the upload fails on because of invalid credentials" in new EmptySite with VerboseLogger with MockAWS {
|
101
101
|
setLocalFile("index.html")
|
102
102
|
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
|
103
103
|
val e = new AmazonServiceException("your credentials are incorrect")
|
@@ -108,14 +108,14 @@ class S3WebsiteSpec extends Specification {
|
|
108
108
|
verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
109
109
|
}
|
110
110
|
|
111
|
-
"try again if the delete fails" in new EmptySite with MockAWS {
|
111
|
+
"try again if the delete fails" in new EmptySite with VerboseLogger with MockAWS {
|
112
112
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
113
113
|
deleteFailsAndThenSucceeds(howManyFailures = 5)
|
114
114
|
Push.pushSite
|
115
115
|
verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
|
116
116
|
}
|
117
117
|
|
118
|
-
"try again if the object listing fails" in new EmptySite with MockAWS {
|
118
|
+
"try again if the object listing fails" in new EmptySite with VerboseLogger with MockAWS {
|
119
119
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
120
120
|
objectListingFailsAndThenSucceeds(howManyFailures = 5)
|
121
121
|
Push.pushSite
|
@@ -124,7 +124,7 @@ class S3WebsiteSpec extends Specification {
|
|
124
124
|
}
|
125
125
|
|
126
126
|
"push with CloudFront" should {
|
127
|
-
"invalidate the updated CloudFront items" in new EmptySite with MockAWS {
|
127
|
+
"invalidate the updated CloudFront items" in new EmptySite with VerboseLogger with MockAWS {
|
128
128
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
129
129
|
setLocalFiles("css/test.css", "articles/index.html")
|
130
130
|
setOutdatedS3Keys("css/test.css", "articles/index.html")
|
@@ -132,14 +132,14 @@ class S3WebsiteSpec extends Specification {
|
|
132
132
|
sentInvalidationRequest.getInvalidationBatch.getPaths.getItems.toSeq.sorted must equalTo(("/css/test.css" :: "/articles/index.html" :: Nil).sorted)
|
133
133
|
}
|
134
134
|
|
135
|
-
"not send CloudFront invalidation requests on new objects" in new EmptySite with MockAWS {
|
135
|
+
"not send CloudFront invalidation requests on new objects" in new EmptySite with VerboseLogger with MockAWS {
|
136
136
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
137
137
|
setLocalFile("newfile.js")
|
138
138
|
Push.pushSite
|
139
139
|
noInvalidationsOccurred must beTrue
|
140
140
|
}
|
141
141
|
|
142
|
-
"not send CloudFront invalidation requests on redirect objects" in new EmptySite with MockAWS {
|
142
|
+
"not send CloudFront invalidation requests on redirect objects" in new EmptySite with VerboseLogger with MockAWS {
|
143
143
|
config = """
|
144
144
|
|cloudfront_distribution_id: EGM1J2JJX9Z
|
145
145
|
|redirects:
|
@@ -149,7 +149,7 @@ class S3WebsiteSpec extends Specification {
|
|
149
149
|
noInvalidationsOccurred must beTrue
|
150
150
|
}
|
151
151
|
|
152
|
-
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new EmptySite with MockAWS {
|
152
|
+
"retry CloudFront responds with TooManyInvalidationsInProgressException" in new EmptySite with VerboseLogger with MockAWS {
|
153
153
|
setTooManyInvalidationsInProgress(4)
|
154
154
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
155
155
|
setLocalFile("test.css")
|
@@ -158,7 +158,7 @@ class S3WebsiteSpec extends Specification {
|
|
158
158
|
sentInvalidationRequests.length must equalTo(4)
|
159
159
|
}
|
160
160
|
|
161
|
-
"retry if CloudFront is temporarily unreachable" in new EmptySite with MockAWS {
|
161
|
+
"retry if CloudFront is temporarily unreachable" in new EmptySite with VerboseLogger with MockAWS {
|
162
162
|
invalidationsFailAndThenSucceed(5)
|
163
163
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
164
164
|
setLocalFile("test.css")
|
@@ -167,7 +167,7 @@ class S3WebsiteSpec extends Specification {
|
|
167
167
|
sentInvalidationRequests.length must equalTo(6)
|
168
168
|
}
|
169
169
|
|
170
|
-
"encode unsafe characters in the keys" in new EmptySite with MockAWS {
|
170
|
+
"encode unsafe characters in the keys" in new EmptySite with VerboseLogger with MockAWS {
|
171
171
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
172
172
|
setLocalFile("articles/arnold's file.html")
|
173
173
|
setOutdatedS3Keys("articles/arnold's file.html")
|
@@ -179,7 +179,7 @@ class S3WebsiteSpec extends Specification {
|
|
179
179
|
* Because CloudFront supports Default Root Objects (http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/DefaultRootObject.html),
|
180
180
|
* we have to guess
|
181
181
|
*/
|
182
|
-
"invalidate the root object '/' if a top-level object is updated or deleted" in new EmptySite with MockAWS {
|
182
|
+
"invalidate the root object '/' if a top-level object is updated or deleted" in new EmptySite with VerboseLogger with MockAWS {
|
183
183
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
184
184
|
setLocalFile("maybe-index.html")
|
185
185
|
setOutdatedS3Keys("maybe-index.html")
|
@@ -189,7 +189,7 @@ class S3WebsiteSpec extends Specification {
|
|
189
189
|
}
|
190
190
|
|
191
191
|
"cloudfront_invalidate_root: true" should {
|
192
|
-
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new EmptySite with MockAWS {
|
192
|
+
"convert CloudFront invalidation paths with the '/index.html' suffix into '/'" in new EmptySite with VerboseLogger with MockAWS {
|
193
193
|
config = """
|
194
194
|
|cloudfront_distribution_id: EGM1J2JJX9Z
|
195
195
|
|cloudfront_invalidate_root: true
|
@@ -202,7 +202,7 @@ class S3WebsiteSpec extends Specification {
|
|
202
202
|
}
|
203
203
|
|
204
204
|
"a site with over 1000 items" should {
|
205
|
-
"split the CloudFront invalidation requests into batches of 1000 items" in new EmptySite with MockAWS {
|
205
|
+
"split the CloudFront invalidation requests into batches of 1000 items" in new EmptySite with VerboseLogger with MockAWS {
|
206
206
|
val files = (1 to 1002).map { i => s"lots-of-files/file-$i"}
|
207
207
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
208
208
|
setLocalFiles(files:_*)
|
@@ -215,18 +215,18 @@ class S3WebsiteSpec extends Specification {
|
|
215
215
|
}
|
216
216
|
|
217
217
|
"push exit status" should {
|
218
|
-
"be 0 all uploads succeed" in new EmptySite with MockAWS {
|
218
|
+
"be 0 all uploads succeed" in new EmptySite with VerboseLogger with MockAWS {
|
219
219
|
setLocalFiles("file.txt")
|
220
220
|
Push.pushSite must equalTo(0)
|
221
221
|
}
|
222
222
|
|
223
|
-
"be 1 if any of the uploads fails" in new EmptySite with MockAWS {
|
223
|
+
"be 1 if any of the uploads fails" in new EmptySite with VerboseLogger with MockAWS {
|
224
224
|
setLocalFiles("file.txt")
|
225
225
|
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
226
226
|
Push.pushSite must equalTo(1)
|
227
227
|
}
|
228
228
|
|
229
|
-
"be 1 if any of the redirects fails" in new EmptySite with MockAWS {
|
229
|
+
"be 1 if any of the redirects fails" in new EmptySite with VerboseLogger with MockAWS {
|
230
230
|
config = """
|
231
231
|
|redirects:
|
232
232
|
| index.php: /index.html
|
@@ -235,14 +235,14 @@ class S3WebsiteSpec extends Specification {
|
|
235
235
|
Push.pushSite must equalTo(1)
|
236
236
|
}
|
237
237
|
|
238
|
-
"be 0 if CloudFront invalidations and uploads succeed"in new EmptySite with MockAWS {
|
238
|
+
"be 0 if CloudFront invalidations and uploads succeed"in new EmptySite with VerboseLogger with MockAWS {
|
239
239
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
240
240
|
setLocalFile("test.css")
|
241
241
|
setOutdatedS3Keys("test.css")
|
242
242
|
Push.pushSite must equalTo(0)
|
243
243
|
}
|
244
244
|
|
245
|
-
"be 1 if CloudFront is unreachable or broken"in new EmptySite with MockAWS {
|
245
|
+
"be 1 if CloudFront is unreachable or broken"in new EmptySite with VerboseLogger with MockAWS {
|
246
246
|
setCloudFrontAsInternallyBroken()
|
247
247
|
config = "cloudfront_distribution_id: EGM1J2JJX9Z"
|
248
248
|
setLocalFile("test.css")
|
@@ -250,19 +250,19 @@ class S3WebsiteSpec extends Specification {
|
|
250
250
|
Push.pushSite must equalTo(1)
|
251
251
|
}
|
252
252
|
|
253
|
-
"be 0 if upload retry succeeds" in new EmptySite with MockAWS {
|
253
|
+
"be 0 if upload retry succeeds" in new EmptySite with VerboseLogger with MockAWS {
|
254
254
|
setLocalFile("index.html")
|
255
255
|
uploadFailsAndThenSucceeds(howManyFailures = 1)
|
256
256
|
Push.pushSite must equalTo(0)
|
257
257
|
}
|
258
258
|
|
259
|
-
"be 1 if delete retry fails" in new EmptySite with MockAWS {
|
259
|
+
"be 1 if delete retry fails" in new EmptySite with VerboseLogger with MockAWS {
|
260
260
|
setLocalFile("index.html")
|
261
261
|
uploadFailsAndThenSucceeds(howManyFailures = 6)
|
262
262
|
Push.pushSite must equalTo(1)
|
263
263
|
}
|
264
264
|
|
265
|
-
"be 1 if an object listing fails" in new EmptySite with MockAWS {
|
265
|
+
"be 1 if an object listing fails" in new EmptySite with VerboseLogger with MockAWS {
|
266
266
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
267
267
|
objectListingFailsAndThenSucceeds(howManyFailures = 6)
|
268
268
|
Push.pushSite must equalTo(1)
|
@@ -270,7 +270,7 @@ class S3WebsiteSpec extends Specification {
|
|
270
270
|
}
|
271
271
|
|
272
272
|
"s3_website.yml file" should {
|
273
|
-
"never be uploaded" in new EmptySite with MockAWS {
|
273
|
+
"never be uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
274
274
|
setLocalFile("s3_website.yml")
|
275
275
|
Push.pushSite
|
276
276
|
noUploadsOccurred must beTrue
|
@@ -278,7 +278,7 @@ class S3WebsiteSpec extends Specification {
|
|
278
278
|
}
|
279
279
|
|
280
280
|
"exclude_from_upload: string" should {
|
281
|
-
"result in matching files not being uploaded" in new EmptySite with MockAWS {
|
281
|
+
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
282
282
|
config = "exclude_from_upload: .DS_.*?"
|
283
283
|
setLocalFile(".DS_Store")
|
284
284
|
Push.pushSite
|
@@ -291,7 +291,7 @@ class S3WebsiteSpec extends Specification {
|
|
291
291
|
- regex
|
292
292
|
- another_exclusion
|
293
293
|
""" should {
|
294
|
-
"result in matching files not being uploaded" in new EmptySite with MockAWS {
|
294
|
+
"result in matching files not being uploaded" in new EmptySite with VerboseLogger with MockAWS {
|
295
295
|
config = """
|
296
296
|
|exclude_from_upload:
|
297
297
|
| - .DS_.*?
|
@@ -304,7 +304,7 @@ class S3WebsiteSpec extends Specification {
|
|
304
304
|
}
|
305
305
|
|
306
306
|
"ignore_on_server: value" should {
|
307
|
-
"not delete the S3 objects that match the ignore value" in new EmptySite with MockAWS {
|
307
|
+
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS {
|
308
308
|
config = "ignore_on_server: logs"
|
309
309
|
setS3Files(S3File("logs/log.txt", ""))
|
310
310
|
Push.pushSite
|
@@ -317,7 +317,7 @@ class S3WebsiteSpec extends Specification {
|
|
317
317
|
- regex
|
318
318
|
- another_ignore
|
319
319
|
""" should {
|
320
|
-
"not delete the S3 objects that match the ignore value" in new EmptySite with MockAWS {
|
320
|
+
"not delete the S3 objects that match the ignore value" in new EmptySite with VerboseLogger with MockAWS {
|
321
321
|
config = """
|
322
322
|
|ignore_on_server:
|
323
323
|
| - .*txt
|
@@ -329,14 +329,14 @@ class S3WebsiteSpec extends Specification {
|
|
329
329
|
}
|
330
330
|
|
331
331
|
"max-age in config" can {
|
332
|
-
"be applied to all files" in new EmptySite with MockAWS {
|
332
|
+
"be applied to all files" in new EmptySite with VerboseLogger with MockAWS {
|
333
333
|
config = "max_age: 60"
|
334
334
|
setLocalFile("index.html")
|
335
335
|
Push.pushSite
|
336
336
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=60")
|
337
337
|
}
|
338
338
|
|
339
|
-
"be applied to files that match the glob" in new EmptySite with MockAWS {
|
339
|
+
"be applied to files that match the glob" in new EmptySite with VerboseLogger with MockAWS {
|
340
340
|
config = """
|
341
341
|
|max_age:
|
342
342
|
| "*.html": 90
|
@@ -346,7 +346,7 @@ class S3WebsiteSpec extends Specification {
|
|
346
346
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
347
347
|
}
|
348
348
|
|
349
|
-
"be applied to directories that match the glob" in new EmptySite with MockAWS {
|
349
|
+
"be applied to directories that match the glob" in new EmptySite with VerboseLogger with MockAWS {
|
350
350
|
config = """
|
351
351
|
|max_age:
|
352
352
|
| "assets/**/*.js": 90
|
@@ -356,7 +356,7 @@ class S3WebsiteSpec extends Specification {
|
|
356
356
|
sentPutObjectRequest.getMetadata.getCacheControl must equalTo("max-age=90")
|
357
357
|
}
|
358
358
|
|
359
|
-
"not be applied if the glob doesn't match" in new EmptySite with MockAWS {
|
359
|
+
"not be applied if the glob doesn't match" in new EmptySite with VerboseLogger with MockAWS {
|
360
360
|
config = """
|
361
361
|
|max_age:
|
362
362
|
| "*.js": 90
|
@@ -366,7 +366,7 @@ class S3WebsiteSpec extends Specification {
|
|
366
366
|
sentPutObjectRequest.getMetadata.getCacheControl must beNull
|
367
367
|
}
|
368
368
|
|
369
|
-
"be used to disable caching" in new EmptySite with MockAWS {
|
369
|
+
"be used to disable caching" in new EmptySite with VerboseLogger with MockAWS {
|
370
370
|
config = "max_age: 0"
|
371
371
|
setLocalFile("index.html")
|
372
372
|
Push.pushSite
|
@@ -375,7 +375,7 @@ class S3WebsiteSpec extends Specification {
|
|
375
375
|
}
|
376
376
|
|
377
377
|
"max-age in config" should {
|
378
|
-
"respect the more specific glob" in new EmptySite with MockAWS {
|
378
|
+
"respect the more specific glob" in new EmptySite with VerboseLogger with MockAWS {
|
379
379
|
config = """
|
380
380
|
|max_age:
|
381
381
|
| "assets/*": 150
|
@@ -389,7 +389,7 @@ class S3WebsiteSpec extends Specification {
|
|
389
389
|
}
|
390
390
|
|
391
391
|
"s3_reduced_redundancy: true in config" should {
|
392
|
-
"result in uploads being marked with reduced redundancy" in new EmptySite with MockAWS {
|
392
|
+
"result in uploads being marked with reduced redundancy" in new EmptySite with VerboseLogger with MockAWS {
|
393
393
|
config = "s3_reduced_redundancy: true"
|
394
394
|
setLocalFile("file.exe")
|
395
395
|
Push.pushSite
|
@@ -398,7 +398,7 @@ class S3WebsiteSpec extends Specification {
|
|
398
398
|
}
|
399
399
|
|
400
400
|
"s3_reduced_redundancy: false in config" should {
|
401
|
-
"result in uploads being marked with the default storage class" in new EmptySite with MockAWS {
|
401
|
+
"result in uploads being marked with the default storage class" in new EmptySite with VerboseLogger with MockAWS {
|
402
402
|
config = "s3_reduced_redundancy: false"
|
403
403
|
setLocalFile("file.exe")
|
404
404
|
Push.pushSite
|
@@ -407,7 +407,7 @@ class S3WebsiteSpec extends Specification {
|
|
407
407
|
}
|
408
408
|
|
409
409
|
"redirect in config" should {
|
410
|
-
"result in a redirect instruction that is sent to AWS" in new EmptySite with MockAWS {
|
410
|
+
"result in a redirect instruction that is sent to AWS" in new EmptySite with VerboseLogger with MockAWS {
|
411
411
|
config = """
|
412
412
|
|redirects:
|
413
413
|
| index.php: /index.html
|
@@ -416,7 +416,7 @@ class S3WebsiteSpec extends Specification {
|
|
416
416
|
sentPutObjectRequest.getRedirectLocation must equalTo("/index.html")
|
417
417
|
}
|
418
418
|
|
419
|
-
"result in max-age=0 Cache-Control header on the object" in new EmptySite with MockAWS {
|
419
|
+
"result in max-age=0 Cache-Control header on the object" in new EmptySite with VerboseLogger with MockAWS {
|
420
420
|
config = """
|
421
421
|
|redirects:
|
422
422
|
| index.php: /index.html
|
@@ -427,7 +427,7 @@ class S3WebsiteSpec extends Specification {
|
|
427
427
|
}
|
428
428
|
|
429
429
|
"redirect in config and an object on the S3 bucket" should {
|
430
|
-
"not result in the S3 object being deleted" in new EmptySite with MockAWS {
|
430
|
+
"not result in the S3 object being deleted" in new EmptySite with VerboseLogger with MockAWS {
|
431
431
|
config = """
|
432
432
|
|redirects:
|
433
433
|
| index.php: /index.html
|
@@ -440,7 +440,7 @@ class S3WebsiteSpec extends Specification {
|
|
440
440
|
}
|
441
441
|
|
442
442
|
"dotfiles" should {
|
443
|
-
"be included in the pushed files" in new EmptySite with MockAWS {
|
443
|
+
"be included in the pushed files" in new EmptySite with VerboseLogger with MockAWS {
|
444
444
|
setLocalFile(".vimrc")
|
445
445
|
Push.pushSite
|
446
446
|
sentPutObjectRequest.getKey must equalTo(".vimrc")
|
@@ -448,25 +448,25 @@ class S3WebsiteSpec extends Specification {
|
|
448
448
|
}
|
449
449
|
|
450
450
|
"content type inference" should {
|
451
|
-
"add charset=utf-8 to all html documents" in new EmptySite with MockAWS {
|
451
|
+
"add charset=utf-8 to all html documents" in new EmptySite with VerboseLogger with MockAWS {
|
452
452
|
setLocalFile("index.html")
|
453
453
|
Push.pushSite
|
454
454
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
455
455
|
}
|
456
456
|
|
457
|
-
"add charset=utf-8 to all text documents" in new EmptySite with MockAWS {
|
457
|
+
"add charset=utf-8 to all text documents" in new EmptySite with VerboseLogger with MockAWS {
|
458
458
|
setLocalFile("index.txt")
|
459
459
|
Push.pushSite
|
460
460
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/plain; charset=utf-8")
|
461
461
|
}
|
462
462
|
|
463
|
-
"add charset=utf-8 to all json documents" in new EmptySite with MockAWS {
|
463
|
+
"add charset=utf-8 to all json documents" in new EmptySite with VerboseLogger with MockAWS {
|
464
464
|
setLocalFile("data.json")
|
465
465
|
Push.pushSite
|
466
466
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("application/json; charset=utf-8")
|
467
467
|
}
|
468
468
|
|
469
|
-
"resolve the content type from file contents" in new EmptySite with MockAWS {
|
469
|
+
"resolve the content type from file contents" in new EmptySite with VerboseLogger with MockAWS {
|
470
470
|
setLocalFileWithContent(("index", "<html><body><h1>hi</h1></body></html>"))
|
471
471
|
Push.pushSite
|
472
472
|
sentPutObjectRequest.getMetadata.getContentType must equalTo("text/html; charset=utf-8")
|
@@ -474,7 +474,7 @@ class S3WebsiteSpec extends Specification {
|
|
474
474
|
}
|
475
475
|
|
476
476
|
"ERB in config file" should {
|
477
|
-
"be evaluated" in new EmptySite with MockAWS {
|
477
|
+
"be evaluated" in new EmptySite with VerboseLogger with MockAWS {
|
478
478
|
config = """
|
479
479
|
|redirects:
|
480
480
|
|<%= ('a'..'f').to_a.map do |t| ' '+t+ ': /'+t+'.html' end.join('\n')%>
|
@@ -484,8 +484,37 @@ class S3WebsiteSpec extends Specification {
|
|
484
484
|
sentPutObjectRequests.forall(_.getRedirectLocation != null) must beTrue
|
485
485
|
}
|
486
486
|
}
|
487
|
+
|
488
|
+
"logging" should {
|
489
|
+
"print the debug messages when --verbose is defined" in new EmptySite with VerboseLogger with MockAWS {
|
490
|
+
Push.pushSite
|
491
|
+
logEntries must contain("[debg] Querying S3 files")
|
492
|
+
}
|
493
|
+
|
494
|
+
"not print the debug messages by default" in new EmptySite with NonVerboseLogger with MockAWS {
|
495
|
+
Push.pushSite
|
496
|
+
logEntries.forall(_.contains("[debg]")) must beFalse
|
497
|
+
}
|
498
|
+
}
|
487
499
|
|
488
500
|
trait MockAWS extends MockS3 with MockCloudFront with Scope
|
501
|
+
|
502
|
+
trait VerboseLogger extends LogCapturer {
|
503
|
+
implicit val logger: Logger = new Logger(verboseOutput = true, logMessage = captureAndPrint)
|
504
|
+
}
|
505
|
+
|
506
|
+
trait NonVerboseLogger extends LogCapturer {
|
507
|
+
implicit val logger: Logger = new Logger(verboseOutput = false, logMessage = captureAndPrint)
|
508
|
+
}
|
509
|
+
|
510
|
+
trait LogCapturer {
|
511
|
+
val logEntries: mutable.Buffer[String] = mutable.Buffer()
|
512
|
+
|
513
|
+
def captureAndPrint(msg: String) {
|
514
|
+
logEntries += msg.replaceAll("\u001B\\[[;\\d]*m", "") // Remove ANSI coloring
|
515
|
+
println(msg)
|
516
|
+
}
|
517
|
+
}
|
489
518
|
|
490
519
|
trait MockCloudFront extends MockAWSHelper {
|
491
520
|
val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
|
@@ -581,7 +610,7 @@ class S3WebsiteSpec extends Specification {
|
|
581
610
|
.listObjects(Matchers.any(classOf[ListObjectsRequest]))
|
582
611
|
}
|
583
612
|
|
584
|
-
def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
|
613
|
+
def asSeenByS3Client(upload: Upload)(implicit config: Config, logger: Logger): PutObjectRequest = {
|
585
614
|
Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
|
586
615
|
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
587
616
|
verify(amazonS3Client).putObject(req.capture())
|
@@ -654,7 +683,8 @@ class S3WebsiteSpec extends Specification {
|
|
654
683
|
|s3_bucket: bucket
|
655
684
|
""".stripMargin
|
656
685
|
|
657
|
-
implicit
|
686
|
+
implicit lazy val testSite: Site = siteWithFilesAndContent(config, localFilesWithContent)
|
687
|
+
implicit def logger: Logger
|
658
688
|
|
659
689
|
def buildSite(
|
660
690
|
config: String = "",
|
@@ -672,7 +702,7 @@ class S3WebsiteSpec extends Specification {
|
|
672
702
|
|$config
|
673
703
|
""".stripMargin
|
674
704
|
)
|
675
|
-
val errorOrSite: Either[ErrorReport, Site] = Site.loadSite(configFile.getAbsolutePath, siteDir.getAbsolutePath)
|
705
|
+
val errorOrSite: Either[ErrorReport, Site] = Site.loadSite(configFile.getAbsolutePath, siteDir.getAbsolutePath)(logger)
|
676
706
|
errorOrSite.left.foreach (error => throw new RuntimeException(error.reportMessage))
|
677
707
|
errorOrSite.right.get
|
678
708
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_website_monadic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.27
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lauri Lehmijoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -115,7 +115,6 @@ files:
|
|
115
115
|
- sbt
|
116
116
|
- src/main/scala/s3/website/CloudFront.scala
|
117
117
|
- src/main/scala/s3/website/Diff.scala
|
118
|
-
- src/main/scala/s3/website/Implicits.scala
|
119
118
|
- src/main/scala/s3/website/Push.scala
|
120
119
|
- src/main/scala/s3/website/Ruby.scala
|
121
120
|
- src/main/scala/s3/website/S3.scala
|