s3_website_monadic 0.0.5 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 64e9b6093bd40e0e6728a66dcfcf5134e5b72d52
4
- data.tar.gz: 6aa1819b0581575150f767fc253f63a635c53a43
3
+ metadata.gz: 9b120ee4e68407bd3c41617d5f2fd2ef96142f3c
4
+ data.tar.gz: 3db76e44515ec798bc4bab1907fa612aefef9e64
5
5
  SHA512:
6
- metadata.gz: 7dcb71a7dd601dca1d6d85f387f97170f7c09eb9e185db2755334af4137f99e42877fffca2ec46ca8995efbaa17ab78fd32dbae866cb91dfbf7b8122263f2f68
7
- data.tar.gz: 3fc944d61d92e04cf7d20532a091a3492c9d22e237c41a659087cb3c8175ff2c023ca8d4fbf59fcf92636007c93fa2b08241f68f21589ab670d7669204004dca
6
+ metadata.gz: 7aa168c9af8a8e74261a76b829300d488e4f7765682504ac8d86cbc963059c11952a6b448e911724d4c5ab52b90eb72f0f278cdf1928485a7146fe558250aab6
7
+ data.tar.gz: 7b1800e6dcfa02bbed5e9b4034f59db180e4864b0dadacebaac526da23e88ecc4a1053addbea5e513e937de613795fe275aaa78634418ab4cebc8e190713a12d
data/s3_website.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "s3_website_monadic"
6
- s.version = "0.0.5"
6
+ s.version = "0.0.6"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Lauri Lehmijoki"]
9
9
  s.email = ["lauri.lehmijoki@iki.fi"]
@@ -2,29 +2,28 @@ package s3.website
2
2
 
3
3
  import s3.website.model.{Redirect, Config}
4
4
  import com.amazonaws.services.cloudfront.{AmazonCloudFrontClient, AmazonCloudFront}
5
- import s3.website.CloudFront.{FailedInvalidation, SuccessfulInvalidation, CloudFrontClientProvider}
6
- import scala.util.{Failure, Success, Try}
5
+ import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider, SuccessfulInvalidation, FailedInvalidation}
6
+ import scala.util.Try
7
7
  import com.amazonaws.services.cloudfront.model.{TooManyInvalidationsInProgressException, Paths, InvalidationBatch, CreateInvalidationRequest}
8
8
  import scala.collection.JavaConversions._
9
9
  import scala.concurrent.duration._
10
- import s3.website.S3.{SuccessfulUpload, PushSuccessReport}
10
+ import s3.website.S3.PushSuccessReport
11
11
  import com.amazonaws.auth.BasicAWSCredentials
12
12
  import s3.website.Logger._
13
13
  import s3.website.S3.SuccessfulUpload
14
14
  import scala.util.Failure
15
- import s3.website.CloudFront.SuccessfulInvalidation
16
15
  import scala.util.Success
17
- import s3.website.CloudFront.FailedInvalidation
18
- import java.net.{URI, URLEncoder}
19
- import java.net.URLEncoder.encode
16
+ import java.net.URI
17
+ import Utils._
20
18
 
21
- class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUnit) {
19
+ class CloudFront(implicit cloudFrontSettings: CloudFrontSettings, config: Config) {
20
+ val cloudFront = cloudFrontSettings.cfClient(config)
22
21
 
23
- def invalidate(invalidationBatch: InvalidationBatch, distributionId: String)(implicit config: Config): InvalidationResult = {
22
+ def invalidate(invalidationBatch: InvalidationBatch, distributionId: String): InvalidationResult = {
24
23
  def tryInvalidate(implicit attempt: Int = 1): Try[SuccessfulInvalidation] =
25
24
  Try {
26
25
  val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
27
- cfClient(config).createInvalidation(invalidationReq)
26
+ cloudFront.createInvalidation(invalidationReq)
28
27
  val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
29
28
  info(result)
30
29
  result
@@ -32,7 +31,7 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
32
31
  case e: TooManyInvalidationsInProgressException =>
33
32
  implicit val duration: Duration = Duration(
34
33
  (fibs drop attempt).head min 15, /* AWS docs way that invalidations complete in 15 minutes */
35
- sleepUnit
34
+ cloudFrontSettings.cloudFrontSleepTimeUnit
36
35
  )
37
36
  pending(maxInvalidationsExceededInfo)
38
37
  Thread.sleep(duration.toMillis)
@@ -63,7 +62,6 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
63
62
 
64
63
  type InvalidationResult = Either[FailedInvalidation, SuccessfulInvalidation]
65
64
 
66
- lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
67
65
  }
68
66
 
69
67
  object CloudFront {
@@ -113,4 +111,9 @@ object CloudFront {
113
111
  }
114
112
  case _ => false
115
113
  }
114
+
115
+ case class CloudFrontSettings(
116
+ cfClient: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
117
+ cloudFrontSleepTimeUnit: TimeUnit = MINUTES
118
+ )
116
119
  }
@@ -18,18 +18,11 @@ import s3.website.model.Update
18
18
  import s3.website.model.NewFile
19
19
  import s3.website.S3.PushSuccessReport
20
20
  import scala.collection.mutable.ArrayBuffer
21
- import s3.website.CloudFront.{CloudFrontClientProvider, toInvalidationBatches}
22
- import s3.website.S3.SuccessfulDelete
23
- import s3.website.CloudFront.SuccessfulInvalidation
24
- import s3.website.S3.SuccessfulUpload
25
- import s3.website.CloudFront.FailedInvalidation
21
+ import s3.website.CloudFront._
26
22
  import s3.website.Logger._
27
23
  import s3.website.S3.SuccessfulDelete
28
24
  import s3.website.CloudFront.SuccessfulInvalidation
29
- import s3.website.S3.SuccessfulUpload
30
- import s3.website.CloudFront.FailedInvalidation
31
- import s3.website.S3.SuccessfulDelete
32
- import s3.website.CloudFront.SuccessfulInvalidation
25
+ import s3.website.S3.S3Settings
33
26
  import s3.website.S3.SuccessfulUpload
34
27
  import s3.website.CloudFront.FailedInvalidation
35
28
 
@@ -38,9 +31,8 @@ object Push {
38
31
  def pushSite(
39
32
  implicit site: Site,
40
33
  executor: ExecutionContextExecutor,
41
- s3ClientProvider: S3ClientProvider = S3.awsS3Client,
42
- cloudFrontClientProvider: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
43
- cloudFrontSleepTimeUnit: TimeUnit = MINUTES
34
+ s3Settings: S3Settings,
35
+ cloudFrontSettings: CloudFrontSettings
44
36
  ): ExitCode = {
45
37
  info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
46
38
  val utils: Utils = new Utils
@@ -54,7 +46,7 @@ object Push {
54
46
  .map { s3File => new S3() delete s3File.s3Key }
55
47
  .map { Right(_) } // To make delete reports type-compatible with upload reports
56
48
  val uploadReports: PushReports = utils toParSeq (redirects.toStream.map(Right(_)) ++ resolveUploads(localFiles, s3Files))
57
- .map { errorOrUpload => errorOrUpload.right.map(new S3() upload ) }
49
+ .map { errorOrUpload => errorOrUpload.right.map(new S3() upload) }
58
50
  uploadReports ++ deleteReports
59
51
  }
60
52
  val errorsOrFinishedPushOps: Either[Error, FinishedPushOperations] = errorsOrReports.right map {
@@ -67,7 +59,7 @@ object Push {
67
59
 
68
60
  def invalidateCloudFrontItems
69
61
  (errorsOrFinishedPushOps: Either[Error, FinishedPushOperations])
70
- (implicit config: Config, cloudFrontClientProvider: CloudFrontClientProvider, cloudFrontSleepTimeUnit: TimeUnit): Option[InvalidationSucceeded] = {
62
+ (implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
71
63
  config.cloudfront_distribution_id.map {
72
64
  distributionId =>
73
65
  val pushSuccessReports = errorsOrFinishedPushOps.fold(
@@ -190,6 +182,8 @@ object Push {
190
182
  implicit site =>
191
183
  val threadPool = newFixedThreadPool(site.config.concurrency_level)
192
184
  implicit val executor = fromExecutor(threadPool)
185
+ implicit val s3Settings = S3Settings()
186
+ implicit val cloudFrontSettings = CloudFrontSettings()
193
187
  val pushStatus = pushSite
194
188
  threadPool.shutdownNow()
195
189
  pushStatus
@@ -10,14 +10,8 @@ import com.amazonaws.AmazonClientException
10
10
  import scala.concurrent.{ExecutionContextExecutor, Future}
11
11
  import s3.website.S3._
12
12
  import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
13
- import s3.website.S3.SuccessfulUpload
14
- import s3.website.S3.FailedUpload
15
- import scala.util.Failure
16
- import scala.Some
17
- import s3.website.model.IOError
18
- import scala.util.Success
19
- import s3.website.model.UserError
20
13
  import s3.website.Logger._
14
+ import s3.website.Utils._
21
15
  import s3.website.S3.SuccessfulUpload
22
16
  import s3.website.S3.SuccessfulDelete
23
17
  import s3.website.S3.FailedUpload
@@ -27,34 +21,50 @@ import s3.website.S3.FailedDelete
27
21
  import s3.website.model.IOError
28
22
  import scala.util.Success
29
23
  import s3.website.model.UserError
24
+ import scala.concurrent.duration.{TimeUnit, Duration}
25
+ import java.util.concurrent.TimeUnit.SECONDS
30
26
 
31
- class S3(implicit s3Client: S3ClientProvider) {
27
+ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
32
28
 
33
- def upload(upload: Upload with UploadTypeResolved)(implicit config: Config, executor: ExecutionContextExecutor): Future[Either[FailedUpload, SuccessfulUpload]] =
29
+ def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): S3PushResult =
34
30
  Future {
35
- s3Client(config) putObject toPutObjectRequest(upload)
31
+ s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
36
32
  val report = SuccessfulUpload(upload)
37
33
  info(report)
38
34
  Right(report)
39
- } recover usingErrorHandler { error =>
40
- FailedUpload(upload.s3Key, error)
41
- }
35
+ } recoverWith retry(
36
+ createReport = error => FailedUpload(upload.s3Key, error),
37
+ retryAction = newAttempt => this.upload(upload)(newAttempt, config)
38
+ )
42
39
 
43
- def delete(s3Key: String)(implicit config: Config, executor: ExecutionContextExecutor): Future[Either[FailedDelete, SuccessfulDelete]] =
40
+ def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): S3PushResult =
44
41
  Future {
45
- s3Client(config) deleteObject(config.s3_bucket, s3Key)
42
+ s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
46
43
  val report = SuccessfulDelete(s3Key)
47
44
  info(report)
48
45
  Right(report)
49
- } recover usingErrorHandler { error =>
50
- FailedDelete(s3Key, error)
51
- }
52
-
53
- def usingErrorHandler[T <: PushFailureReport, F <: PushFailureReport](f: (Throwable) => T): PartialFunction[Throwable, Either[T, F]] = {
54
- case error =>
55
- val report = f(error)
56
- info(report)
57
- Left(report)
46
+ } recoverWith retry(
47
+ createReport = error => FailedDelete(s3Key, error),
48
+ retryAction = newAttempt => this.delete(s3Key)(newAttempt, config)
49
+ )
50
+
51
+ type S3PushResult = Future[Either[PushFailureReport, PushSuccessReport]]
52
+
53
+ type Attempt = Int
54
+
55
+ def retry(createReport: (Throwable) => PushFailureReport, retryAction: (Attempt) => S3PushResult)(implicit attempt: Attempt):
56
+ PartialFunction[Throwable, S3PushResult] = {
57
+ case error: Throwable =>
58
+ val failureReport = createReport(error)
59
+ if (attempt == 6) {
60
+ info(failureReport)
61
+ Future(Left(failureReport))
62
+ } else {
63
+ val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
64
+ pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
65
+ Thread.sleep(sleepDuration.toMillis)
66
+ retryAction(attempt + 1)
67
+ }
58
68
  }
59
69
 
60
70
  def toPutObjectRequest(upload: Upload)(implicit config: Config) =
@@ -89,7 +99,7 @@ class S3(implicit s3Client: S3ClientProvider) {
89
99
  object S3 {
90
100
  def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
91
101
 
92
- def resolveS3Files(implicit config: Config, s3ClientProvider: S3ClientProvider): Either[Error, Stream[S3File]] = Try {
102
+ def resolveS3Files(implicit config: Config, s3Settings: S3Settings): Either[Error, Stream[S3File]] = Try {
93
103
  objectSummaries()
94
104
  } match {
95
105
  case Success(remoteFiles) =>
@@ -100,8 +110,8 @@ object S3 {
100
110
  Left(IOError(error))
101
111
  }
102
112
 
103
- def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3ClientProvider: S3ClientProvider): Stream[S3File] = {
104
- val objects: ObjectListing = s3ClientProvider(config).listObjects({
113
+ def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3Settings: S3Settings): Stream[S3File] = {
114
+ val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
105
115
  val req = new ListObjectsRequest()
106
116
  req.setBucketName(config.s3_bucket)
107
117
  nextMarker.foreach(req.setMarker)
@@ -143,4 +153,9 @@ object S3 {
143
153
  }
144
154
 
145
155
  type S3ClientProvider = (Config) => AmazonS3
156
+
157
+ case class S3Settings(
158
+ s3Client: S3ClientProvider = S3.awsS3Client,
159
+ retrySleepTimeUnit: TimeUnit = SECONDS
160
+ )
146
161
  }
@@ -12,6 +12,10 @@ class Utils(implicit config: Config) {
12
12
  }
13
13
  }
14
14
 
15
+ object Utils {
16
+ lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
17
+ }
18
+
15
19
  object Logger {
16
20
  import Rainbow._
17
21
  def info(msg: String) = println(s"[${"info".blue}] $msg")
@@ -14,13 +14,13 @@ import com.amazonaws.services.s3.model._
14
14
  import scala.concurrent.ExecutionContext.Implicits.global
15
15
  import scala.concurrent.Await
16
16
  import scala.concurrent.duration._
17
- import s3.website.S3.S3ClientProvider
17
+ import s3.website.S3.{S3Settings, S3ClientProvider}
18
18
  import scala.collection.JavaConversions._
19
19
  import s3.website.model.NewFile
20
20
  import scala.Some
21
21
  import com.amazonaws.AmazonServiceException
22
22
  import org.apache.commons.codec.digest.DigestUtils.md5Hex
23
- import s3.website.CloudFront.CloudFrontClientProvider
23
+ import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider}
24
24
  import com.amazonaws.services.cloudfront.AmazonCloudFront
25
25
  import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
26
26
  import org.mockito.stubbing.Answer
@@ -93,6 +93,21 @@ class S3WebsiteSpec extends Specification {
93
93
  Push.pushSite
94
94
  sentDelete must equalTo("old.html")
95
95
  }
96
+
97
+ "try again if the upload fails" in new SiteDirectory with MockAWS {
98
+ implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
99
+ uploadFailsAndThenSucceeds(howManyFailures = 5)
100
+ Push.pushSite
101
+ verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
102
+ }
103
+
104
+ "try again if the delete fails" in new SiteDirectory with MockAWS {
105
+ implicit val site = buildSite()
106
+ setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
107
+ deleteFailsAndThenSucceeds(howManyFailures = 5)
108
+ Push.pushSite
109
+ verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
110
+ }
96
111
  }
97
112
 
98
113
  "push with CloudFront" should {
@@ -185,6 +200,18 @@ class S3WebsiteSpec extends Specification {
185
200
  )
186
201
  Push.pushSite must equalTo(1)
187
202
  }
203
+
204
+ "be 0 if upload retry succeeds" in new SiteDirectory with MockAWS {
205
+ implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
206
+ uploadFailsAndThenSucceeds(howManyFailures = 1)
207
+ Push.pushSite must equalTo(0)
208
+ }
209
+
210
+ "be 1 if delete retry fails" in new SiteDirectory with MockAWS {
211
+ implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
212
+ uploadFailsAndThenSucceeds(howManyFailures = 6)
213
+ Push.pushSite must equalTo(1)
214
+ }
188
215
  }
189
216
 
190
217
  "s3_website.yml file" should {
@@ -323,8 +350,10 @@ class S3WebsiteSpec extends Specification {
323
350
 
324
351
  trait MockCloudFront {
325
352
  val amazonCloudFrontClient = mock(classOf[AmazonCloudFront])
326
- implicit val cfClientProvider: CloudFrontClientProvider = _ => amazonCloudFrontClient
327
- implicit val cloudFrontSleepTimeUnit: TimeUnit = MILLISECONDS
353
+ implicit val cfSettings: CloudFrontSettings = CloudFrontSettings(
354
+ cfClient = _ => amazonCloudFrontClient,
355
+ cloudFrontSleepTimeUnit = MICROSECONDS
356
+ )
328
357
 
329
358
  def sentInvalidationRequests: Seq[CreateInvalidationRequest] = {
330
359
  val createInvalidationReq = ArgumentCaptor.forClass(classOf[CreateInvalidationRequest])
@@ -359,7 +388,10 @@ class S3WebsiteSpec extends Specification {
359
388
 
360
389
  trait MockS3 {
361
390
  val amazonS3Client = mock(classOf[AmazonS3])
362
- implicit val s3ClientProvider: S3ClientProvider = _ => amazonS3Client
391
+ implicit val s3Settings: S3Settings = S3Settings(
392
+ s3Client = _ => amazonS3Client,
393
+ retrySleepTimeUnit = MICROSECONDS
394
+ )
363
395
  val s3ObjectListing = new ObjectListing
364
396
  when(amazonS3Client.listObjects(Matchers.any(classOf[ListObjectsRequest]))).thenReturn(s3ObjectListing)
365
397
 
@@ -376,6 +408,32 @@ class S3WebsiteSpec extends Specification {
376
408
 
377
409
  val s3 = new S3()
378
410
 
411
+ def uploadFailsAndThenSucceeds(howManyFailures: Int) {
412
+ var callCount = 0
413
+ doAnswer(new Answer[PutObjectResult] {
414
+ override def answer(invocation: InvocationOnMock) = {
415
+ callCount += 1
416
+ if (callCount <= howManyFailures)
417
+ throw new AmazonServiceException("AWS is temporarily down")
418
+ else
419
+ mock(classOf[PutObjectResult])
420
+ }
421
+ }).when(amazonS3Client).putObject(Matchers.anyObject())
422
+ }
423
+
424
+ def deleteFailsAndThenSucceeds(howManyFailures: Int) {
425
+ var callCount = 0
426
+ doAnswer(new Answer[DeleteObjectRequest] {
427
+ override def answer(invocation: InvocationOnMock) = {
428
+ callCount += 1
429
+ if (callCount <= howManyFailures)
430
+ throw new AmazonServiceException("AWS is temporarily down")
431
+ else
432
+ mock(classOf[DeleteObjectRequest])
433
+ }
434
+ }).when(amazonS3Client).deleteObject(Matchers.anyString(), Matchers.anyString())
435
+ }
436
+
379
437
  def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
380
438
  Await.ready(s3.upload(upload withUploadType NewFile), Duration("1 s"))
381
439
  val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3_website_monadic
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Lauri Lehmijoki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-05-06 00:00:00.000000000 Z
11
+ date: 2014-05-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk