s3_website_monadic 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/s3_website.gemspec +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +15 -12
- data/src/main/scala/s3/website/Push.scala +8 -14
- data/src/main/scala/s3/website/S3.scala +42 -27
- data/src/main/scala/s3/website/Utils.scala +4 -0
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +63 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9b120ee4e68407bd3c41617d5f2fd2ef96142f3c
|
4
|
+
data.tar.gz: 3db76e44515ec798bc4bab1907fa612aefef9e64
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7aa168c9af8a8e74261a76b829300d488e4f7765682504ac8d86cbc963059c11952a6b448e911724d4c5ab52b90eb72f0f278cdf1928485a7146fe558250aab6
|
7
|
+
data.tar.gz: 7b1800e6dcfa02bbed5e9b4034f59db180e4864b0dadacebaac526da23e88ecc4a1053addbea5e513e937de613795fe275aaa78634418ab4cebc8e190713a12d
|
data/s3_website.gemspec
CHANGED
@@ -2,29 +2,28 @@ package s3.website
|
|
2
2
|
|
3
3
|
import s3.website.model.{Redirect, Config}
|
4
4
|
import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
|
5
|
-
import s3.website.CloudFront.{
|
6
|
-
import scala.util.
|
5
|
+
import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider, SuccessfulInvalidation, FailedInvalidation}
|
6
|
+
import scala.util.Try
|
7
7
|
import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
|
8
8
|
import scala.collection.JavaConversions._
|
9
9
|
import scala.concurrent.duration._
|
10
|
-
import s3.website.S3.
|
10
|
+
import s3.website.S3.PushSuccessReport
|
11
11
|
import com.amazonaws.auth.BasicAWSCredentials
|
12
12
|
import s3.website.Logger._
|
13
13
|
import s3.website.S3.SuccessfulUpload
|
14
14
|
import scala.util.Failure
|
15
|
-
import s3.website.CloudFront.SuccessfulInvalidation
|
16
15
|
import scala.util.Success
|
17
|
-
import
|
18
|
-
import
|
19
|
-
import java.net.URLEncoder.encode
|
16
|
+
import java.net.URI
|
17
|
+
import Utils._
|
20
18
|
|
21
|
-
class CloudFront(implicit
|
19
|
+
class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config) {
|
20
|
+
val cloudFront = cloudFrontSettings.cfClient(config)
|
22
21
|
|
23
|
-
def invalidate(invalidationBatch: InvalidationBatch, distributionId: String)
|
22
|
+
def invalidate(invalidationBatch: InvalidationBatch, distributionId: String): InvalidationResult = {
|
24
23
|
def tryInvalidate(implicit attempt: Int = 1): Try[SuccessfulInvalidation] =
|
25
24
|
Try {
|
26
25
|
val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
|
27
|
-
|
26
|
+
cloudFront.createInvalidation(invalidationReq)
|
28
27
|
val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
|
29
28
|
info(result)
|
30
29
|
result
|
@@ -32,7 +31,7 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
|
|
32
31
|
case e: TooManyInvalidationsInProgressException =>
|
33
32
|
implicit val duration: Duration = Duration(
|
34
33
|
(fibs drop attempt).head min 15, /* AWS docs way that invalidations complete in 15 minutes */
|
35
|
-
|
34
|
+
cloudFrontSettings.cloudFrontSleepTimeUnit
|
36
35
|
)
|
37
36
|
pending(maxInvalidationsExceededInfo)
|
38
37
|
Thread.sleep(duration.toMillis)
|
@@ -63,7 +62,6 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
|
|
63
62
|
|
64
63
|
type InvalidationResult = Either[FailedInvalidation, SuccessfulInvalidation]
|
65
64
|
|
66
|
-
lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
|
67
65
|
}
|
68
66
|
|
69
67
|
object CloudFront {
|
@@ -113,4 +111,9 @@ object CloudFront {
|
|
113
111
|
}
|
114
112
|
case _ => false
|
115
113
|
}
|
114
|
+
|
115
|
+
case class CloudFrontSettings(
|
116
|
+
cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
|
117
|
+
cloudFrontSleepTimeUnit: TimeUnit = MINUTES
|
118
|
+
)
|
116
119
|
}
|
@@ -18,18 +18,11 @@ import s3.website.model.Update
|
|
18
18
|
import s3.website.model.NewFile
|
19
19
|
import s3.website.S3.PushSuccessReport
|
20
20
|
import scala.collection.mutable.ArrayBuffer
|
21
|
-
import s3.website.CloudFront.
|
22
|
-
import s3.website.S3.SuccessfulDelete
|
23
|
-
import s3.website.CloudFront.SuccessfulInvalidation
|
24
|
-
import s3.website.S3.SuccessfulUpload
|
25
|
-
import s3.website.CloudFront.FailedInvalidation
|
21
|
+
import s3.website.CloudFront._
|
26
22
|
import s3.website.Logger._
|
27
23
|
import s3.website.S3.SuccessfulDelete
|
28
24
|
import s3.website.CloudFront.SuccessfulInvalidation
|
29
|
-
import s3.website.S3.
|
30
|
-
import s3.website.CloudFront.FailedInvalidation
|
31
|
-
import s3.website.S3.SuccessfulDelete
|
32
|
-
import s3.website.CloudFront.SuccessfulInvalidation
|
25
|
+
import s3.website.S3.S3Settings
|
33
26
|
import s3.website.S3.SuccessfulUpload
|
34
27
|
import s3.website.CloudFront.FailedInvalidation
|
35
28
|
|
@@ -38,9 +31,8 @@ object Push {
|
|
38
31
|
def pushSite(
|
39
32
|
implicit site: Site,
|
40
33
|
executor: ExecutionContextExecutor,
|
41
|
-
|
42
|
-
|
43
|
-
cloudFrontSleepTimeUnit: TimeUnit = MINUTES
|
34
|
+
s3Settings: S3Settings,
|
35
|
+
cloudFrontSettings: CloudFrontSettings
|
44
36
|
): ExitCode = {
|
45
37
|
info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
46
38
|
val utils: Utils = new Utils
|
@@ -54,7 +46,7 @@ object Push {
|
|
54
46
|
.map { s3File => new S3() delete s3File.s3Key }
|
55
47
|
.map { Right(_) } // To make delete reports type-compatible with upload reports
|
56
48
|
val uploadReports: PushReports = utils toParSeq (redirects.toStream.map(Right(_)) ++ resolveUploads(localFiles, s3Files))
|
57
|
-
.map { errorOrUpload => errorOrUpload.right.map(new S3() upload
|
49
|
+
.map { errorOrUpload => errorOrUpload.right.map(new S3() upload) }
|
58
50
|
uploadReports ++ deleteReports
|
59
51
|
}
|
60
52
|
val errorsOrFinishedPushOps: Either[Error, FinishedPushOperations] = errorsOrReports.right map {
|
@@ -67,7 +59,7 @@ object Push {
|
|
67
59
|
|
68
60
|
def invalidateCloudFrontItems
|
69
61
|
(errorsOrFinishedPushOps: Either[Error, FinishedPushOperations])
|
70
|
-
(implicit config: Config,
|
62
|
+
(implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
|
71
63
|
config.cloudfront_distribution_id.map {
|
72
64
|
distributionId =>
|
73
65
|
val pushSuccessReports = errorsOrFinishedPushOps.fold(
|
@@ -190,6 +182,8 @@ object Push {
|
|
190
182
|
implicit site =>
|
191
183
|
val threadPool = newFixedThreadPool(site.config.concurrency_level)
|
192
184
|
implicit val executor = fromExecutor(threadPool)
|
185
|
+
implicit val s3Settings = S3Settings()
|
186
|
+
implicit val cloudFrontSettings = CloudFrontSettings()
|
193
187
|
val pushStatus = pushSite
|
194
188
|
threadPool.shutdownNow()
|
195
189
|
pushStatus
|
@@ -10,14 +10,8 @@ import com.amazonaws.AmazonClientException
|
|
10
10
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
11
11
|
import s3.website.S3._
|
12
12
|
import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
|
13
|
-
import s3.website.S3.SuccessfulUpload
|
14
|
-
import s3.website.S3.FailedUpload
|
15
|
-
import scala.util.Failure
|
16
|
-
import scala.Some
|
17
|
-
import s3.website.model.IOError
|
18
|
-
import scala.util.Success
|
19
|
-
import s3.website.model.UserError
|
20
13
|
import s3.website.Logger._
|
14
|
+
import s3.website.Utils._
|
21
15
|
import s3.website.S3.SuccessfulUpload
|
22
16
|
import s3.website.S3.SuccessfulDelete
|
23
17
|
import s3.website.S3.FailedUpload
|
@@ -27,34 +21,50 @@ import s3.website.S3.FailedDelete
|
|
27
21
|
import s3.website.model.IOError
|
28
22
|
import scala.util.Success
|
29
23
|
import s3.website.model.UserError
|
24
|
+
import scala.concurrent.duration.{TimeUnit, Duration}
|
25
|
+
import java.util.concurrent.TimeUnit.SECONDS
|
30
26
|
|
31
|
-
class S3(implicit
|
27
|
+
class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
|
32
28
|
|
33
|
-
def upload(upload: Upload with UploadTypeResolved)(implicit
|
29
|
+
def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): S3PushResult =
|
34
30
|
Future {
|
35
|
-
s3Client(config) putObject toPutObjectRequest(upload)
|
31
|
+
s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
|
36
32
|
val report = SuccessfulUpload(upload)
|
37
33
|
info(report)
|
38
34
|
Right(report)
|
39
|
-
}
|
40
|
-
FailedUpload(upload.s3Key, error)
|
41
|
-
|
35
|
+
} recoverWith retry(
|
36
|
+
createReport = error => FailedUpload(upload.s3Key, error),
|
37
|
+
retryAction = newAttempt => this.upload(upload)(newAttempt, config)
|
38
|
+
)
|
42
39
|
|
43
|
-
def delete(s3Key: String)(implicit
|
40
|
+
def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): S3PushResult =
|
44
41
|
Future {
|
45
|
-
s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
42
|
+
s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
46
43
|
val report = SuccessfulDelete(s3Key)
|
47
44
|
info(report)
|
48
45
|
Right(report)
|
49
|
-
}
|
50
|
-
FailedDelete(s3Key, error)
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
46
|
+
} recoverWith retry(
|
47
|
+
createReport = error => FailedDelete(s3Key, error),
|
48
|
+
retryAction = newAttempt => this.delete(s3Key)(newAttempt, config)
|
49
|
+
)
|
50
|
+
|
51
|
+
type S3PushResult = Future[Either[PushFailureReport, PushSuccessReport]]
|
52
|
+
|
53
|
+
type Attempt = Int
|
54
|
+
|
55
|
+
def retry(createReport: (Throwable) => PushFailureReport, retryAction: (Attempt) => S3PushResult)(implicit attempt: Attempt):
|
56
|
+
PartialFunction[Throwable, S3PushResult] = {
|
57
|
+
case error: Throwable =>
|
58
|
+
val failureReport = createReport(error)
|
59
|
+
if (attempt == 6) {
|
60
|
+
info(failureReport)
|
61
|
+
Future(Left(failureReport))
|
62
|
+
} else {
|
63
|
+
val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
|
64
|
+
pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
|
65
|
+
Thread.sleep(sleepDuration.toMillis)
|
66
|
+
retryAction(attempt + 1)
|
67
|
+
}
|
58
68
|
}
|
59
69
|
|
60
70
|
def toPutObjectRequest(upload: Upload)(implicit config: Config) =
|
@@ -89,7 +99,7 @@ class S3(implicit s3Client: S3ClientProvider) {
|
|
89
99
|
object S3 {
|
90
100
|
def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
|
91
101
|
|
92
|
-
def resolveS3Files(implicit config: Config,
|
102
|
+
def resolveS3Files(implicit config: Config, s3Settings: S3Settings): Either[Error, Stream[S3File]] = Try {
|
93
103
|
objectSummaries()
|
94
104
|
} match {
|
95
105
|
case Success(remoteFiles) =>
|
@@ -100,8 +110,8 @@ object S3 {
|
|
100
110
|
Left(IOError(error))
|
101
111
|
}
|
102
112
|
|
103
|
-
def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config,
|
104
|
-
val objects: ObjectListing =
|
113
|
+
def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3Settings: S3Settings): Stream[S3File] = {
|
114
|
+
val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
|
105
115
|
val req = new ListObjectsRequest()
|
106
116
|
req.setBucketName(config.s3_bucket)
|
107
117
|
nextMarker.foreach(req.setMarker)
|
@@ -143,4 +153,9 @@ object S3 {
|
|
143
153
|
}
|
144
154
|
|
145
155
|
type S3ClientProvider = (Config) => AmazonS3
|
156
|
+
|
157
|
+
case class S3Settings(
|
158
|
+
s3Client: S3ClientProvider = S3.awsS3Client,
|
159
|
+
retrySleepTimeUnit: TimeUnit = SECONDS
|
160
|
+
)
|
146
161
|
}
|
@@ -12,6 +12,10 @@ class Utils(implicit config: Config) {
|
|
12
12
|
}
|
13
13
|
}
|
14
14
|
|
15
|
+
object Utils {
|
16
|
+
lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
|
17
|
+
}
|
18
|
+
|
15
19
|
object Logger {
|
16
20
|
import Rainbow._
|
17
21
|
def info(msg: String) = println(s"[${"info".blue}] $msg")
|
@@ -14,13 +14,13 @@ import com.amazonaws.services.s3.model._
|
|
14
14
|
import scala.concurrent.ExecutionContext.Implicits.global
|
15
15
|
import scala.concurrent.Await
|
16
16
|
import scala.concurrent.duration._
|
17
|
-
import s3.website.S3.S3ClientProvider
|
17
|
+
import s3.website.S3.{S3Settings, S3ClientProvider}
|
18
18
|
import scala.collection.JavaConversions._
|
19
19
|
import s3.website.model.NewFile
|
20
20
|
import scala.Some
|
21
21
|
import com.amazonaws.AmazonServiceException
|
22
22
|
import org.apache.commons.codec.digest.DigestUtils.md5Hex
|
23
|
-
import s3.website.CloudFront.CloudFrontClientProvider
|
23
|
+
import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider}
|
24
24
|
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
25
25
|
import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
|
26
26
|
import org.mockito.stubbing.Answer
|
@@ -93,6 +93,21 @@ class S3WebsiteSpec extends Specification {
|
|
93
93
|
Push.pushSite
|
94
94
|
sentDelete must equalTo("old.html")
|
95
95
|
}
|
96
|
+
|
97
|
+
"try again if the upload fails" in new SiteDirectory with MockAWS {
|
98
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
99
|
+
uploadFailsAndThenSucceeds(howManyFailures = 5)
|
100
|
+
Push.pushSite
|
101
|
+
verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
|
102
|
+
}
|
103
|
+
|
104
|
+
"try again if the delete fails" in new SiteDirectory with MockAWS {
|
105
|
+
implicit val site = buildSite()
|
106
|
+
setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
|
107
|
+
deleteFailsAndThenSucceeds(howManyFailures = 5)
|
108
|
+
Push.pushSite
|
109
|
+
verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
|
110
|
+
}
|
96
111
|
}
|
97
112
|
|
98
113
|
"push with CloudFront" should {
|
@@ -185,6 +200,18 @@ class S3WebsiteSpec extends Specification {
|
|
185
200
|
)
|
186
201
|
Push.pushSite must equalTo(1)
|
187
202
|
}
|
203
|
+
|
204
|
+
"be 0 if upload retry succeeds" in new SiteDirectory with MockAWS {
|
205
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
206
|
+
uploadFailsAndThenSucceeds(howManyFailures = 1)
|
207
|
+
Push.pushSite must equalTo(0)
|
208
|
+
}
|
209
|
+
|
210
|
+
"be 1 if delete retry fails" in new SiteDirectory with MockAWS {
|
211
|
+
implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
|
212
|
+
uploadFailsAndThenSucceeds(howManyFailures = 6)
|
213
|
+
Push.pushSite must equalTo(1)
|
214
|
+
}
|
188
215
|
}
|
189
216
|
|
190
217
|
"s3_website.yml file" should {
|
@@ -323,8 +350,10 @@ class S3WebsiteSpec extends Specification {
|
|
323
350
|
|
324
351
|
trait MockCloudFront {
|
325
352
|
val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
|
326
|
-
implicit val
|
327
|
-
|
353
|
+
implicit val cfSettings: CloudFrontSettings = CloudFrontSettings(
|
354
|
+
cfClient = _ => amazonCloudFrontClient,
|
355
|
+
cloudFrontSleepTimeUnit = MICROSECONDS
|
356
|
+
)
|
328
357
|
|
329
358
|
def sentInvalidationRequests: Seq[CreateInvalidationRequest] = {
|
330
359
|
val createInvalidationReq = ArgumentCaptor.forClass(classOf[CreateInvalidationRequest])
|
@@ -359,7 +388,10 @@ class S3WebsiteSpec extends Specification {
|
|
359
388
|
|
360
389
|
trait MockS3 {
|
361
390
|
val amazonS3Client = mock(classOf[AmazonS3])
|
362
|
-
implicit val
|
391
|
+
implicit val s3Settings: S3Settings = S3Settings(
|
392
|
+
s3Client = _ => amazonS3Client,
|
393
|
+
retrySleepTimeUnit = MICROSECONDS
|
394
|
+
)
|
363
395
|
val s3ObjectListing = new ObjectListing
|
364
396
|
when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
|
365
397
|
|
@@ -376,6 +408,32 @@ class S3WebsiteSpec extends Specification {
|
|
376
408
|
|
377
409
|
val s3 = new S3()
|
378
410
|
|
411
|
+
def uploadFailsAndThenSucceeds(howManyFailures: Int) {
|
412
|
+
var callCount = 0
|
413
|
+
doAnswer(new Answer[PutObjectResult] {
|
414
|
+
override def answer(invocation: InvocationOnMock) = {
|
415
|
+
callCount += 1
|
416
|
+
if (callCount <= howManyFailures)
|
417
|
+
throw new AmazonServiceException("AWS is temporarily down")
|
418
|
+
else
|
419
|
+
mock(classOf[PutObjectResult])
|
420
|
+
}
|
421
|
+
}).when(amazonS3Client).putObject(Matchers.anyObject())
|
422
|
+
}
|
423
|
+
|
424
|
+
def deleteFailsAndThenSucceeds(howManyFailures: Int) {
|
425
|
+
var callCount = 0
|
426
|
+
doAnswer(new Answer[DeleteObjectRequest] {
|
427
|
+
override def answer(invocation: InvocationOnMock) = {
|
428
|
+
callCount += 1
|
429
|
+
if (callCount <= howManyFailures)
|
430
|
+
throw new AmazonServiceException("AWS is temporarily down")
|
431
|
+
else
|
432
|
+
mock(classOf[DeleteObjectRequest])
|
433
|
+
}
|
434
|
+
}).when(amazonS3Client).deleteObject(Matchers.anyString(), Matchers.anyString())
|
435
|
+
}
|
436
|
+
|
379
437
|
def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
|
380
438
|
Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
|
381
439
|
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_website_monadic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lauri Lehmijoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: aws-sdk
|