s3_website_monadic 0.0.6 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/s3_website.gemspec +1 -1
- data/src/main/scala/s3/website/Diff.scala +3 -2
- data/src/main/scala/s3/website/Push.scala +17 -14
- data/src/main/scala/s3/website/S3.scala +39 -49
- data/src/main/scala/s3/website/model/Config.scala +19 -14
- data/src/main/scala/s3/website/model/Site.scala +3 -2
- data/src/main/scala/s3/website/model/errors.scala +10 -5
- data/src/main/scala/s3/website/model/push.scala +3 -2
- data/src/main/scala/s3/website/package.scala +2 -0
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +59 -23
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 714e16805c804d5b86d6097a85ba4ec85a4dee14
|
4
|
+
data.tar.gz: fd750b7d7de92913e893104d5945cfb78184d511
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 403cb29f93d49f16f36dd52e72f4f7f6fd897137f8b485cd7940eedbc59940528b054a051b971e9e35098155dfb4afc79cc3f284b15b6395736a7bd250152d42
|
7
|
+
data.tar.gz: 1c10547952fa577f6d574d6b08aad9849a3746a723952c1262a53db05c27dcb0851c682cbc505630964840494bf513337c2b6c448292417d6bf4f73d4a14a930
|
data/s3_website.gemspec
CHANGED
@@ -2,6 +2,7 @@ package s3.website
|
|
2
2
|
|
3
3
|
import s3.website.model._
|
4
4
|
import s3.website.Ruby.rubyRegexMatches
|
5
|
+
import s3.website._
|
5
6
|
|
6
7
|
object Diff {
|
7
8
|
|
@@ -17,7 +18,7 @@ object Diff {
|
|
17
18
|
}
|
18
19
|
|
19
20
|
def resolveUploads(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
|
20
|
-
Stream[Either[
|
21
|
+
Stream[Either[ErrorReport, Upload with UploadTypeResolved]] = {
|
21
22
|
val remoteS3KeysIndex = s3Files.map(_.s3Key).toSet
|
22
23
|
val remoteMd5Index = s3Files.map(_.md5).toSet
|
23
24
|
localFiles
|
@@ -36,7 +37,7 @@ object Diff {
|
|
36
37
|
def isUpdate(remoteS3KeysIndex: Set[String], remoteMd5Index: Set[String])(u: Upload) =
|
37
38
|
remoteS3KeysIndex.exists(_ == u.s3Key) && !remoteMd5Index.exists(remoteMd5 => u.essence.right.exists(_.md5 == remoteMd5))
|
38
39
|
|
39
|
-
def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[
|
40
|
+
def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] =
|
40
41
|
for (upload <- LocalFile.toUpload(localFile).right)
|
41
42
|
yield upload
|
42
43
|
}
|
@@ -23,6 +23,7 @@ import s3.website.Logger._
|
|
23
23
|
import s3.website.S3.SuccessfulDelete
|
24
24
|
import s3.website.CloudFront.SuccessfulInvalidation
|
25
25
|
import s3.website.S3.S3Settings
|
26
|
+
import s3.website.CloudFront.CloudFrontSettings
|
26
27
|
import s3.website.S3.SuccessfulUpload
|
27
28
|
import s3.website.CloudFront.FailedInvalidation
|
28
29
|
|
@@ -37,19 +38,21 @@ object Push {
|
|
37
38
|
info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
38
39
|
val utils: Utils = new Utils
|
39
40
|
|
41
|
+
val redirects = Redirect.resolveRedirects
|
42
|
+
val redirectResults = redirects.map(new S3() upload)
|
43
|
+
|
40
44
|
val errorsOrReports = for {
|
41
|
-
s3Files <- resolveS3Files.right
|
45
|
+
s3Files <- Await.result(resolveS3Files(), 1 minutes).right
|
42
46
|
localFiles <- resolveLocalFiles.right
|
43
47
|
} yield {
|
44
|
-
val redirects = Redirect.resolveRedirects
|
45
48
|
val deleteReports: PushReports = utils toParSeq resolveDeletes(localFiles, s3Files, redirects)
|
46
49
|
.map { s3File => new S3() delete s3File.s3Key }
|
47
50
|
.map { Right(_) } // To make delete reports type-compatible with upload reports
|
48
|
-
val uploadReports: PushReports = utils toParSeq
|
49
|
-
.map {
|
50
|
-
uploadReports ++ deleteReports
|
51
|
+
val uploadReports: PushReports = utils toParSeq resolveUploads(localFiles, s3Files)
|
52
|
+
.map { _.right.map(new S3() upload) }
|
53
|
+
uploadReports ++ deleteReports ++ redirectResults.map(Right(_))
|
51
54
|
}
|
52
|
-
val errorsOrFinishedPushOps: Either[
|
55
|
+
val errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations] = errorsOrReports.right map {
|
53
56
|
uploadReports => awaitForUploads(uploadReports)
|
54
57
|
}
|
55
58
|
val invalidationSucceeded = invalidateCloudFrontItems(errorsOrFinishedPushOps)
|
@@ -58,7 +61,7 @@ object Push {
|
|
58
61
|
}
|
59
62
|
|
60
63
|
def invalidateCloudFrontItems
|
61
|
-
(errorsOrFinishedPushOps: Either[
|
64
|
+
(errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
|
62
65
|
(implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
|
63
66
|
config.cloudfront_distribution_id.map {
|
64
67
|
distributionId =>
|
@@ -91,17 +94,17 @@ object Push {
|
|
91
94
|
|
92
95
|
type InvalidationSucceeded = Boolean
|
93
96
|
|
94
|
-
def afterPushFinished(errorsOrFinishedUploads: Either[
|
97
|
+
def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])(implicit config: Config): ExitCode = {
|
95
98
|
errorsOrFinishedUploads.right.foreach { finishedUploads =>
|
96
99
|
val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
|
97
100
|
info(s"Summary: $pushCounts")
|
98
101
|
}
|
99
|
-
errorsOrFinishedUploads.left foreach (err => fail(s"Encountered error: ${err.
|
102
|
+
errorsOrFinishedUploads.left foreach (err => fail(s"Encountered an error: ${err.reportMessage}"))
|
100
103
|
val exitCode = errorsOrFinishedUploads.fold(
|
101
104
|
_ => 1,
|
102
105
|
finishedUploads => finishedUploads.foldLeft(0) { (memo, finishedUpload) =>
|
103
106
|
memo + finishedUpload.fold(
|
104
|
-
(error:
|
107
|
+
(error: ErrorReport) => 1,
|
105
108
|
(failedOrSucceededUpload: Either[PushFailureReport, PushSuccessReport]) =>
|
106
109
|
if (failedOrSucceededUpload.isLeft) 1 else 0
|
107
110
|
)
|
@@ -124,7 +127,7 @@ object Push {
|
|
124
127
|
|
125
128
|
def resolvePushCounts(implicit finishedOperations: FinishedPushOperations) = finishedOperations.foldLeft(PushCounts()) {
|
126
129
|
(counts: PushCounts, uploadReport) => uploadReport.fold(
|
127
|
-
(error:
|
130
|
+
(error: ErrorReport) => counts.copy(failures = counts.failures + 1),
|
128
131
|
failureOrSuccess => failureOrSuccess.fold(
|
129
132
|
(failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
|
130
133
|
(successReport: PushSuccessReport) => successReport match {
|
@@ -161,8 +164,8 @@ object Push {
|
|
161
164
|
redirects: Int = 0,
|
162
165
|
deletes: Int = 0
|
163
166
|
)
|
164
|
-
type FinishedPushOperations = ParSeq[Either[
|
165
|
-
type PushReports = ParSeq[Either[
|
167
|
+
type FinishedPushOperations = ParSeq[Either[ErrorReport, Either[PushFailureReport, PushSuccessReport]]]
|
168
|
+
type PushReports = ParSeq[Either[ErrorReport, Future[Either[PushFailureReport, PushSuccessReport]]]]
|
166
169
|
case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
|
167
170
|
type ExitCode = Int
|
168
171
|
|
@@ -188,7 +191,7 @@ object Push {
|
|
188
191
|
threadPool.shutdownNow()
|
189
192
|
pushStatus
|
190
193
|
}
|
191
|
-
errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.
|
194
|
+
errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.reportMessage}"))
|
192
195
|
System.exit(errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus))
|
193
196
|
}
|
194
197
|
}
|
@@ -5,68 +5,45 @@ import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
|
|
5
5
|
import com.amazonaws.auth.BasicAWSCredentials
|
6
6
|
import com.amazonaws.services.s3.model._
|
7
7
|
import scala.collection.JavaConversions._
|
8
|
-
import scala.util.Try
|
9
|
-
import com.amazonaws.AmazonClientException
|
10
8
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
11
9
|
import s3.website.S3._
|
12
10
|
import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
|
13
11
|
import s3.website.Logger._
|
14
12
|
import s3.website.Utils._
|
13
|
+
import scala.concurrent.duration.{Duration, TimeUnit}
|
14
|
+
import java.util.concurrent.TimeUnit.SECONDS
|
15
15
|
import s3.website.S3.SuccessfulUpload
|
16
16
|
import s3.website.S3.SuccessfulDelete
|
17
17
|
import s3.website.S3.FailedUpload
|
18
|
-
import scala.util.Failure
|
19
18
|
import scala.Some
|
20
19
|
import s3.website.S3.FailedDelete
|
21
|
-
import s3.website.
|
22
|
-
import
|
23
|
-
import s3.website.model.UserError
|
24
|
-
import scala.concurrent.duration.{TimeUnit, Duration}
|
25
|
-
import java.util.concurrent.TimeUnit.SECONDS
|
20
|
+
import s3.website.S3.S3Settings
|
21
|
+
import s3.website.model.Error.isClientError
|
26
22
|
|
27
23
|
class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
|
28
24
|
|
29
|
-
def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config):
|
25
|
+
def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
|
30
26
|
Future {
|
31
27
|
s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
|
32
28
|
val report = SuccessfulUpload(upload)
|
33
29
|
info(report)
|
34
30
|
Right(report)
|
35
31
|
} recoverWith retry(
|
36
|
-
|
32
|
+
createFailureReport = error => FailedUpload(upload.s3Key, error),
|
37
33
|
retryAction = newAttempt => this.upload(upload)(newAttempt, config)
|
38
34
|
)
|
39
35
|
|
40
|
-
def delete(s3Key: String)(implicit a: Attempt = 1, config: Config):
|
36
|
+
def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
|
41
37
|
Future {
|
42
38
|
s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
43
39
|
val report = SuccessfulDelete(s3Key)
|
44
40
|
info(report)
|
45
41
|
Right(report)
|
46
42
|
} recoverWith retry(
|
47
|
-
|
43
|
+
createFailureReport = error => FailedDelete(s3Key, error),
|
48
44
|
retryAction = newAttempt => this.delete(s3Key)(newAttempt, config)
|
49
45
|
)
|
50
46
|
|
51
|
-
type S3PushResult = Future[Either[PushFailureReport, PushSuccessReport]]
|
52
|
-
|
53
|
-
type Attempt = Int
|
54
|
-
|
55
|
-
def retry(createReport: (Throwable) => PushFailureReport, retryAction: (Attempt) => S3PushResult)(implicit attempt: Attempt):
|
56
|
-
PartialFunction[Throwable, S3PushResult] = {
|
57
|
-
case error: Throwable =>
|
58
|
-
val failureReport = createReport(error)
|
59
|
-
if (attempt == 6) {
|
60
|
-
info(failureReport)
|
61
|
-
Future(Left(failureReport))
|
62
|
-
} else {
|
63
|
-
val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
|
64
|
-
pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
65
|
-
Thread.sleep(sleepDuration.toMillis)
|
66
|
-
retryAction(attempt + 1)
|
67
|
-
}
|
68
|
-
}
|
69
|
-
|
70
47
|
def toPutObjectRequest(upload: Upload)(implicit config: Config) =
|
71
48
|
upload.essence.fold(
|
72
49
|
redirect => {
|
@@ -99,36 +76,49 @@ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
|
|
99
76
|
object S3 {
|
100
77
|
def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
|
101
78
|
|
102
|
-
def resolveS3Files(
|
103
|
-
|
104
|
-
|
105
|
-
case Success(remoteFiles) =>
|
106
|
-
Right(remoteFiles)
|
107
|
-
case Failure(error) if error.isInstanceOf[AmazonClientException] =>
|
108
|
-
Left(UserError(error.getMessage))
|
109
|
-
case Failure(error) =>
|
110
|
-
Left(IOError(error))
|
111
|
-
}
|
112
|
-
|
113
|
-
def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3Settings: S3Settings): Stream[S3File] = {
|
79
|
+
def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil)
|
80
|
+
(implicit attempt: Attempt = 1, config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor): ObjectListingResult = Future {
|
81
|
+
nextMarker.foreach(m => info(s"Fetching the next part of the object listing from S3 (starting from $m)"))
|
114
82
|
val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
|
115
83
|
val req = new ListObjectsRequest()
|
116
84
|
req.setBucketName(config.s3_bucket)
|
117
85
|
nextMarker.foreach(req.setMarker)
|
118
86
|
req
|
119
87
|
})
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
}
|
88
|
+
objects
|
89
|
+
} flatMap { objects =>
|
90
|
+
val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_)))
|
91
|
+
Option(objects.getNextMarker)
|
92
|
+
.fold(Future(Right(s3Files)): ObjectListingResult)(nextMarker => resolveS3Files(Some(nextMarker), s3Files))
|
93
|
+
} recoverWith retry(
|
94
|
+
createFailureReport = error => UserError(s"Failed to fetch an object listing (${error.getMessage})"),
|
95
|
+
retryAction = nextAttempt => resolveS3Files(nextMarker, alreadyResolved)(nextAttempt, config, s3Settings, ec)
|
96
|
+
)
|
97
|
+
|
98
|
+
type ObjectListingResult = Future[Either[ErrorReport, Seq[S3File]]]
|
126
99
|
|
127
100
|
sealed trait PushFailureReport extends FailureReport
|
128
101
|
sealed trait PushSuccessReport extends SuccessReport {
|
129
102
|
def s3Key: String
|
130
103
|
}
|
131
104
|
|
105
|
+
def retry[L <: Report, R](createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
|
106
|
+
(implicit attempt: Attempt, s3Settings: S3Settings, ec: ExecutionContextExecutor):
|
107
|
+
PartialFunction[Throwable, Future[Either[L, R]]] = {
|
108
|
+
case error: Throwable if attempt == 6 || isClientError(error) =>
|
109
|
+
val failureReport = createFailureReport(error)
|
110
|
+
fail(failureReport.reportMessage)
|
111
|
+
Future(Left(failureReport))
|
112
|
+
case error: Throwable =>
|
113
|
+
val failureReport = createFailureReport(error)
|
114
|
+
val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
|
115
|
+
pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
116
|
+
Thread.sleep(sleepDuration.toMillis)
|
117
|
+
retryAction(attempt + 1)
|
118
|
+
}
|
119
|
+
|
120
|
+
type Attempt = Int
|
121
|
+
|
132
122
|
case class SuccessfulUpload(upload: Upload with UploadTypeResolved) extends PushSuccessReport {
|
133
123
|
def reportMessage =
|
134
124
|
upload.uploadType match {
|
@@ -3,6 +3,7 @@ package s3.website.model
|
|
3
3
|
import scala.util.{Failure, Try}
|
4
4
|
import scala.collection.JavaConversions._
|
5
5
|
import s3.website.Ruby.rubyRuntime
|
6
|
+
import s3.website.ErrorReport
|
6
7
|
|
7
8
|
case class Config(
|
8
9
|
s3_id: String,
|
@@ -22,7 +23,7 @@ case class Config(
|
|
22
23
|
)
|
23
24
|
|
24
25
|
object Config {
|
25
|
-
def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[
|
26
|
+
def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {
|
26
27
|
val yamlValue = for {
|
27
28
|
optionalValue <- loadOptionalValue(key)
|
28
29
|
} yield {
|
@@ -35,7 +36,7 @@ object Config {
|
|
35
36
|
yamlValue getOrElse Left(UserError(s"The key $key has to have a boolean or [string] value"))
|
36
37
|
}
|
37
38
|
|
38
|
-
def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[
|
39
|
+
def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, Seq[String]]]] = {
|
39
40
|
val yamlValue = for {
|
40
41
|
valueOption <- loadOptionalValue(key)
|
41
42
|
} yield {
|
@@ -48,7 +49,7 @@ object Config {
|
|
48
49
|
yamlValue getOrElse Left(UserError(s"The key $key has to have a string or [string] value"))
|
49
50
|
}
|
50
51
|
|
51
|
-
def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[
|
52
|
+
def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, Map[String, Int]]]] = {
|
52
53
|
val key = "max_age"
|
53
54
|
val yamlValue = for {
|
54
55
|
maxAgeOption <- loadOptionalValue(key)
|
@@ -62,7 +63,7 @@ object Config {
|
|
62
63
|
yamlValue getOrElse Left(UserError(s"The key $key has to have an int or (string -> int) value"))
|
63
64
|
}
|
64
65
|
|
65
|
-
def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[
|
66
|
+
def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
|
66
67
|
loadOptionalString("s3_endpoint").right flatMap { endpointString =>
|
67
68
|
endpointString.map(S3Endpoint.forString) match {
|
68
69
|
case Some(Right(endpoint)) => Right(Some(endpoint))
|
@@ -71,17 +72,17 @@ object Config {
|
|
71
72
|
}
|
72
73
|
}
|
73
74
|
|
74
|
-
def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[
|
75
|
+
def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[String, String]]] = {
|
75
76
|
val key = "redirects"
|
76
77
|
val yamlValue = for {
|
77
78
|
redirectsOption <- loadOptionalValue(key)
|
78
79
|
redirects <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
|
79
80
|
} yield Right(redirects)
|
80
81
|
|
81
|
-
yamlValue getOrElse Left(UserError(s"The key $key has to have
|
82
|
+
yamlValue getOrElse Left(UserError(s"The key $key has to have a (string -> string) value"))
|
82
83
|
}
|
83
84
|
|
84
|
-
def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[
|
85
|
+
def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
|
85
86
|
val yamlValue = for {
|
86
87
|
valueOption <- loadOptionalValue(key)
|
87
88
|
stringValue <- Try(valueOption.asInstanceOf[Option[String]].get)
|
@@ -94,7 +95,7 @@ object Config {
|
|
94
95
|
}
|
95
96
|
}
|
96
97
|
|
97
|
-
def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[
|
98
|
+
def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[String]] = {
|
98
99
|
val yamlValueOption = for {
|
99
100
|
valueOption <- loadOptionalValue(key)
|
100
101
|
optionalString <- Try(valueOption.asInstanceOf[Option[String]])
|
@@ -107,7 +108,7 @@ object Config {
|
|
107
108
|
}
|
108
109
|
}
|
109
110
|
|
110
|
-
def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[
|
111
|
+
def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Boolean]] = {
|
111
112
|
val yamlValueOption = for {
|
112
113
|
valueOption <- loadOptionalValue(key)
|
113
114
|
optionalBoolean <- Try(valueOption.asInstanceOf[Option[Boolean]])
|
@@ -120,7 +121,7 @@ object Config {
|
|
120
121
|
}
|
121
122
|
}
|
122
123
|
|
123
|
-
def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[
|
124
|
+
def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Int]] = {
|
124
125
|
val yamlValueOption = for {
|
125
126
|
valueOption <- loadOptionalValue(key)
|
126
127
|
optionalInt <- Try(valueOption.asInstanceOf[Option[Int]])
|
@@ -139,12 +140,16 @@ object Config {
|
|
139
140
|
}
|
140
141
|
|
141
142
|
def erbEval(erbString: String): Try[String] = Try {
|
142
|
-
val erbStringWithoutComments = erbString.replaceAll("
|
143
|
+
val erbStringWithoutComments = erbString.replaceAll("^\\s*#.*", "")
|
143
144
|
rubyRuntime.evalScriptlet(
|
144
145
|
s"""
|
145
|
-
require 'erb'
|
146
|
-
|
147
|
-
|
146
|
+
|require 'erb'
|
147
|
+
|
|
148
|
+
|str = <<-ERBSTR
|
149
|
+
|$erbStringWithoutComments
|
150
|
+
|ERBSTR
|
151
|
+
|ERB.new(str).result
|
152
|
+
""".stripMargin
|
148
153
|
).asJavaString()
|
149
154
|
}
|
150
155
|
|
@@ -10,13 +10,14 @@ import s3.website.Logger._
|
|
10
10
|
import scala.util.Failure
|
11
11
|
import s3.website.model.Config.UnsafeYaml
|
12
12
|
import scala.util.Success
|
13
|
+
import s3.website.ErrorReport
|
13
14
|
|
14
15
|
case class Site(rootDirectory: String, config: Config) {
|
15
16
|
def resolveS3Key(file: File) = file.getAbsolutePath.replace(rootDirectory, "").replaceFirst("^/", "")
|
16
17
|
}
|
17
18
|
|
18
19
|
object Site {
|
19
|
-
def loadSite(yamlConfigPath: String, siteRootDirectory: String): Either[
|
20
|
+
def loadSite(yamlConfigPath: String, siteRootDirectory: String): Either[ErrorReport, Site] = {
|
20
21
|
val yamlObjectTry = for {
|
21
22
|
yamlString <- Try(fromFile(new File(yamlConfigPath)).mkString)
|
22
23
|
yamlWithErbEvaluated <- erbEval(yamlString)
|
@@ -25,7 +26,7 @@ object Site {
|
|
25
26
|
yamlObjectTry match {
|
26
27
|
case Success(yamlObject) =>
|
27
28
|
implicit val unsafeYaml = UnsafeYaml(yamlObject)
|
28
|
-
val config: Either[
|
29
|
+
val config: Either[ErrorReport, Config] = for {
|
29
30
|
s3_id <- loadRequiredString("s3_id").right
|
30
31
|
s3_secret <- loadRequiredString("s3_secret").right
|
31
32
|
s3_bucket <- loadRequiredString("s3_bucket").right
|
@@ -1,11 +1,16 @@
|
|
1
1
|
package s3.website.model
|
2
2
|
|
3
|
-
|
4
|
-
|
3
|
+
import com.amazonaws.AmazonServiceException
|
4
|
+
import com.amazonaws.AmazonServiceException.ErrorType.Client
|
5
|
+
import s3.website.ErrorReport
|
6
|
+
|
7
|
+
object Error {
|
8
|
+
def isClientError(error: Throwable) =
|
9
|
+
error.isInstanceOf[AmazonServiceException] && error.asInstanceOf[AmazonServiceException].getErrorType == Client
|
5
10
|
}
|
6
11
|
|
7
|
-
case class UserError(
|
12
|
+
case class UserError(reportMessage: String) extends ErrorReport
|
8
13
|
|
9
|
-
case class IOError(exception: Throwable) extends
|
10
|
-
def
|
14
|
+
case class IOError(exception: Throwable) extends ErrorReport {
|
15
|
+
def reportMessage = exception.getMessage
|
11
16
|
}
|
@@ -9,6 +9,7 @@ import java.util.zip.GZIPOutputStream
|
|
9
9
|
import org.apache.commons.io.IOUtils
|
10
10
|
import org.apache.tika.Tika
|
11
11
|
import s3.website.Ruby._
|
12
|
+
import s3.website._
|
12
13
|
import s3.website.model.Encoding.Gzip
|
13
14
|
import scala.util.Failure
|
14
15
|
import scala.Some
|
@@ -59,7 +60,7 @@ case class LocalFile(
|
|
59
60
|
) extends S3KeyProvider
|
60
61
|
|
61
62
|
object LocalFile {
|
62
|
-
def toUpload(localFile: LocalFile)(implicit config: Config): Either[
|
63
|
+
def toUpload(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] = Try {
|
63
64
|
def fis(file: File): InputStream = new FileInputStream(file)
|
64
65
|
def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
|
65
66
|
val sourceFile: File = localFile
|
@@ -107,7 +108,7 @@ object LocalFile {
|
|
107
108
|
|
108
109
|
lazy val tika = new Tika()
|
109
110
|
|
110
|
-
def resolveLocalFiles(implicit site: Site): Either[
|
111
|
+
def resolveLocalFiles(implicit site: Site): Either[ErrorReport, Seq[LocalFile]] = Try {
|
111
112
|
val files = recursiveListFiles(new File(site.rootDirectory)).filterNot(_.isDirectory)
|
112
113
|
files map { file =>
|
113
114
|
val s3Key = site.resolveS3Key(file)
|
@@ -18,13 +18,15 @@ import s3.website.S3.{S3Settings, S3ClientProvider}
|
|
18
18
|
import scala.collection.JavaConversions._
|
19
19
|
import s3.website.model.NewFile
|
20
20
|
import scala.Some
|
21
|
-
import com.amazonaws.AmazonServiceException
|
21
|
+
import com.amazonaws.{AmazonClientException, AmazonServiceException}
|
22
22
|
import org.apache.commons.codec.digest.DigestUtils.md5Hex
|
23
23
|
import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider}
|
24
24
|
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
25
25
|
import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
|
26
26
|
import org.mockito.stubbing.Answer
|
27
27
|
import org.mockito.invocation.InvocationOnMock
|
28
|
+
import com.amazonaws.AmazonServiceException.ErrorType
|
29
|
+
import java.util.concurrent.atomic.AtomicInteger
|
28
30
|
|
29
31
|
class S3WebsiteSpec extends Specification {
|
30
32
|
|
@@ -101,6 +103,17 @@ class S3WebsiteSpec extends Specification {
|
|
101
103
|
verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
102
104
|
}
|
103
105
|
|
106
|
+
"not try again if the upload fails on because of the client" in new SiteDirectory with MockAWS {
|
107
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
108
|
+
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
|
109
|
+
val e = new AmazonServiceException("your credentials are incorrect")
|
110
|
+
e.setErrorType(ErrorType.Client)
|
111
|
+
e
|
112
|
+
}
|
113
|
+
Push.pushSite
|
114
|
+
verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
115
|
+
}
|
116
|
+
|
104
117
|
"try again if the delete fails" in new SiteDirectory with MockAWS {
|
105
118
|
implicit val site = buildSite()
|
106
119
|
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
@@ -108,6 +121,14 @@ class S3WebsiteSpec extends Specification {
|
|
108
121
|
Push.pushSite
|
109
122
|
verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
|
110
123
|
}
|
124
|
+
|
125
|
+
"try again if the object listing fails" in new SiteDirectory with MockAWS {
|
126
|
+
implicit val site = buildSite()
|
127
|
+
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
128
|
+
objectListingFailsAndThenSucceeds(howManyFailures = 5)
|
129
|
+
Push.pushSite
|
130
|
+
verify(amazonS3Client, times(6)).listObjects(Matchers.any(classOf[ListObjectsRequest]))
|
131
|
+
}
|
111
132
|
}
|
112
133
|
|
113
134
|
"push with CloudFront" should {
|
@@ -184,6 +205,12 @@ class S3WebsiteSpec extends Specification {
|
|
184
205
|
Push.pushSite must equalTo(1)
|
185
206
|
}
|
186
207
|
|
208
|
+
"be 1 if any of the redirects fails" in new SiteDirectory with MockAWS {
|
209
|
+
implicit val site = buildSite(defaultConfig.copy(redirects = Some(Map("index.php" -> "/index.html"))))
|
210
|
+
when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
|
211
|
+
Push.pushSite must equalTo(1)
|
212
|
+
}
|
213
|
+
|
187
214
|
"be 0 if CloudFront invalidations and uploads succeed"in new SiteDirectory with MockAWS {
|
188
215
|
implicit val site = siteWithFiles(
|
189
216
|
config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
|
@@ -212,6 +239,13 @@ class S3WebsiteSpec extends Specification {
|
|
212
239
|
uploadFailsAndThenSucceeds(howManyFailures = 6)
|
213
240
|
Push.pushSite must equalTo(1)
|
214
241
|
}
|
242
|
+
|
243
|
+
"be 1 if an object listing fails" in new SiteDirectory with MockAWS {
|
244
|
+
implicit val site = buildSite()
|
245
|
+
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
246
|
+
objectListingFailsAndThenSucceeds(howManyFailures = 6)
|
247
|
+
Push.pushSite must equalTo(1)
|
248
|
+
}
|
215
249
|
}
|
216
250
|
|
217
251
|
"s3_website.yml file" should {
|
@@ -408,30 +442,32 @@ class S3WebsiteSpec extends Specification {
|
|
408
442
|
|
409
443
|
val s3 = new S3()
|
410
444
|
|
411
|
-
def uploadFailsAndThenSucceeds(howManyFailures: Int) {
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
callCount += 1
|
416
|
-
if (callCount <= howManyFailures)
|
417
|
-
throw new AmazonServiceException("AWS is temporarily down")
|
418
|
-
else
|
419
|
-
mock(classOf[PutObjectResult])
|
420
|
-
}
|
421
|
-
}).when(amazonS3Client).putObject(Matchers.anyObject())
|
445
|
+
def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
446
|
+
doAnswer(temporaryFailure(classOf[PutObjectResult]))
|
447
|
+
.when(amazonS3Client)
|
448
|
+
.putObject(Matchers.anyObject())
|
422
449
|
}
|
423
450
|
|
424
|
-
def deleteFailsAndThenSucceeds(howManyFailures: Int) {
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
451
|
+
def deleteFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
452
|
+
doAnswer(temporaryFailure(classOf[DeleteObjectRequest]))
|
453
|
+
.when(amazonS3Client)
|
454
|
+
.deleteObject(Matchers.anyString(), Matchers.anyString())
|
455
|
+
}
|
456
|
+
|
457
|
+
def objectListingFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
|
458
|
+
doAnswer(temporaryFailure(classOf[ObjectListing]))
|
459
|
+
.when(amazonS3Client)
|
460
|
+
.listObjects(Matchers.any(classOf[ListObjectsRequest]))
|
461
|
+
}
|
462
|
+
|
463
|
+
def temporaryFailure[T](clazz: Class[T])(implicit callCount: AtomicInteger, howManyFailures: Int) = new Answer[T] {
|
464
|
+
def answer(invocation: InvocationOnMock) = {
|
465
|
+
callCount.incrementAndGet()
|
466
|
+
if (callCount.get() <= howManyFailures)
|
467
|
+
throw new AmazonServiceException("AWS is temporarily down")
|
468
|
+
else
|
469
|
+
mock(clazz)
|
470
|
+
}
|
435
471
|
}
|
436
472
|
|
437
473
|
def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_website_monadic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lauri Lehmijoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|