s3_website_monadic 0.0.6 → 0.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9b120ee4e68407bd3c41617d5f2fd2ef96142f3c
4
- data.tar.gz: 3db76e44515ec798bc4bab1907fa612aefef9e64
3
+ metadata.gz: 714e16805c804d5b86d6097a85ba4ec85a4dee14
4
+ data.tar.gz: fd750b7d7de92913e893104d5945cfb78184d511
5
5
  SHA512:
6
- metadata.gz: 7aa168c9af8a8e74261a76b829300d488e4f7765682504ac8d86cbc963059c11952a6b448e911724d4c5ab52b90eb72f0f278cdf1928485a7146fe558250aab6
7
- data.tar.gz: 7b1800e6dcfa02bbed5e9b4034f59db180e4864b0dadacebaac526da23e88ecc4a1053addbea5e513e937de613795fe275aaa78634418ab4cebc8e190713a12d
6
+ metadata.gz: 403cb29f93d49f16f36dd52e72f4f7f6fd897137f8b485cd7940eedbc59940528b054a051b971e9e35098155dfb4afc79cc3f284b15b6395736a7bd250152d42
7
+ data.tar.gz: 1c10547952fa577f6d574d6b08aad9849a3746a723952c1262a53db05c27dcb0851c682cbc505630964840494bf513337c2b6c448292417d6bf4f73d4a14a930
data/s3_website.gemspec CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "s3_website_monadic"
6
- s.version = "0.0.6"
6
+ s.version = "0.0.7"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Lauri Lehmijoki"]
9
9
  s.email = ["lauri.lehmijoki@iki.fi"]
@@ -2,6 +2,7 @@ package s3.website
2
2
 
3
3
  import s3.website.model._
4
4
  import s3.website.Ruby.rubyRegexMatches
5
+ import s3.website._
5
6
 
6
7
  object Diff {
7
8
 
@@ -17,7 +18,7 @@ object Diff {
17
18
  }
18
19
 
19
20
  def resolveUploads(localFiles: Seq[LocalFile], s3Files: Seq[S3File])(implicit config: Config):
20
- Stream[Either[Error, Upload with UploadTypeResolved]] = {
21
+ Stream[Either[ErrorReport, Upload with UploadTypeResolved]] = {
21
22
  val remoteS3KeysIndex = s3Files.map(_.s3Key).toSet
22
23
  val remoteMd5Index = s3Files.map(_.md5).toSet
23
24
  localFiles
@@ -36,7 +37,7 @@ object Diff {
36
37
  def isUpdate(remoteS3KeysIndex: Set[String], remoteMd5Index: Set[String])(u: Upload) =
37
38
  remoteS3KeysIndex.exists(_ == u.s3Key) && !remoteMd5Index.exists(remoteMd5 => u.essence.right.exists(_.md5 == remoteMd5))
38
39
 
39
- def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[Error, Upload] =
40
+ def resolveUploadSource(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] =
40
41
  for (upload <- LocalFile.toUpload(localFile).right)
41
42
  yield upload
42
43
  }
@@ -23,6 +23,7 @@ import s3.website.Logger._
23
23
  import s3.website.S3.SuccessfulDelete
24
24
  import s3.website.CloudFront.SuccessfulInvalidation
25
25
  import s3.website.S3.S3Settings
26
+ import s3.website.CloudFront.CloudFrontSettings
26
27
  import s3.website.S3.SuccessfulUpload
27
28
  import s3.website.CloudFront.FailedInvalidation
28
29
 
@@ -37,19 +38,21 @@ object Push {
37
38
  info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
38
39
  val utils: Utils = new Utils
39
40
 
41
+ val redirects = Redirect.resolveRedirects
42
+ val redirectResults = redirects.map(new S3() upload)
43
+
40
44
  val errorsOrReports = for {
41
- s3Files <- resolveS3Files.right
45
+ s3Files <- Await.result(resolveS3Files(), 1 minutes).right
42
46
  localFiles <- resolveLocalFiles.right
43
47
  } yield {
44
- val redirects = Redirect.resolveRedirects
45
48
  val deleteReports: PushReports = utils toParSeq resolveDeletes(localFiles, s3Files, redirects)
46
49
  .map { s3File => new S3() delete s3File.s3Key }
47
50
  .map { Right(_) } // To make delete reports type-compatible with upload reports
48
- val uploadReports: PushReports = utils toParSeq (redirects.toStream.map(Right(_)) ++ resolveUploads(localFiles, s3Files))
49
- .map { errorOrUpload => errorOrUpload.right.map(new S3() upload) }
50
- uploadReports ++ deleteReports
51
+ val uploadReports: PushReports = utils toParSeq resolveUploads(localFiles, s3Files)
52
+ .map { _.right.map(new S3() upload) }
53
+ uploadReports ++ deleteReports ++ redirectResults.map(Right(_))
51
54
  }
52
- val errorsOrFinishedPushOps: Either[Error, FinishedPushOperations] = errorsOrReports.right map {
55
+ val errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations] = errorsOrReports.right map {
53
56
  uploadReports => awaitForUploads(uploadReports)
54
57
  }
55
58
  val invalidationSucceeded = invalidateCloudFrontItems(errorsOrFinishedPushOps)
@@ -58,7 +61,7 @@ object Push {
58
61
  }
59
62
 
60
63
  def invalidateCloudFrontItems
61
- (errorsOrFinishedPushOps: Either[Error, FinishedPushOperations])
64
+ (errorsOrFinishedPushOps: Either[ErrorReport, FinishedPushOperations])
62
65
  (implicit config: Config, cloudFrontSettings: CloudFrontSettings): Option[InvalidationSucceeded] = {
63
66
  config.cloudfront_distribution_id.map {
64
67
  distributionId =>
@@ -91,17 +94,17 @@ object Push {
91
94
 
92
95
  type InvalidationSucceeded = Boolean
93
96
 
94
- def afterPushFinished(errorsOrFinishedUploads: Either[Error, FinishedPushOperations], invalidationSucceeded: Option[Boolean])(implicit config: Config): ExitCode = {
97
+ def afterPushFinished(errorsOrFinishedUploads: Either[ErrorReport, FinishedPushOperations], invalidationSucceeded: Option[Boolean])(implicit config: Config): ExitCode = {
95
98
  errorsOrFinishedUploads.right.foreach { finishedUploads =>
96
99
  val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
97
100
  info(s"Summary: $pushCounts")
98
101
  }
99
- errorsOrFinishedUploads.left foreach (err => fail(s"Encountered error: ${err.message}"))
102
+ errorsOrFinishedUploads.left foreach (err => fail(s"Encountered an error: ${err.reportMessage}"))
100
103
  val exitCode = errorsOrFinishedUploads.fold(
101
104
  _ => 1,
102
105
  finishedUploads => finishedUploads.foldLeft(0) { (memo, finishedUpload) =>
103
106
  memo + finishedUpload.fold(
104
- (error: Error) => 1,
107
+ (error: ErrorReport) => 1,
105
108
  (failedOrSucceededUpload: Either[PushFailureReport, PushSuccessReport]) =>
106
109
  if (failedOrSucceededUpload.isLeft) 1 else 0
107
110
  )
@@ -124,7 +127,7 @@ object Push {
124
127
 
125
128
  def resolvePushCounts(implicit finishedOperations: FinishedPushOperations) = finishedOperations.foldLeft(PushCounts()) {
126
129
  (counts: PushCounts, uploadReport) => uploadReport.fold(
127
- (error: model.Error) => counts.copy(failures = counts.failures + 1),
130
+ (error: ErrorReport) => counts.copy(failures = counts.failures + 1),
128
131
  failureOrSuccess => failureOrSuccess.fold(
129
132
  (failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
130
133
  (successReport: PushSuccessReport) => successReport match {
@@ -161,8 +164,8 @@ object Push {
161
164
  redirects: Int = 0,
162
165
  deletes: Int = 0
163
166
  )
164
- type FinishedPushOperations = ParSeq[Either[model.Error, Either[PushFailureReport, PushSuccessReport]]]
165
- type PushReports = ParSeq[Either[model.Error, Future[Either[PushFailureReport, PushSuccessReport]]]]
167
+ type FinishedPushOperations = ParSeq[Either[ErrorReport, Either[PushFailureReport, PushSuccessReport]]]
168
+ type PushReports = ParSeq[Either[ErrorReport, Future[Either[PushFailureReport, PushSuccessReport]]]]
166
169
  case class PushResult(threadPool: ExecutorService, uploadReports: PushReports)
167
170
  type ExitCode = Int
168
171
 
@@ -188,7 +191,7 @@ object Push {
188
191
  threadPool.shutdownNow()
189
192
  pushStatus
190
193
  }
191
- errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.message}"))
194
+ errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.reportMessage}"))
192
195
  System.exit(errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus))
193
196
  }
194
197
  }
@@ -5,68 +5,45 @@ import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
5
5
  import com.amazonaws.auth.BasicAWSCredentials
6
6
  import com.amazonaws.services.s3.model._
7
7
  import scala.collection.JavaConversions._
8
- import scala.util.Try
9
- import com.amazonaws.AmazonClientException
10
8
  import scala.concurrent.{ExecutionContextExecutor, Future}
11
9
  import s3.website.S3._
12
10
  import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
13
11
  import s3.website.Logger._
14
12
  import s3.website.Utils._
13
+ import scala.concurrent.duration.{Duration, TimeUnit}
14
+ import java.util.concurrent.TimeUnit.SECONDS
15
15
  import s3.website.S3.SuccessfulUpload
16
16
  import s3.website.S3.SuccessfulDelete
17
17
  import s3.website.S3.FailedUpload
18
- import scala.util.Failure
19
18
  import scala.Some
20
19
  import s3.website.S3.FailedDelete
21
- import s3.website.model.IOError
22
- import scala.util.Success
23
- import s3.website.model.UserError
24
- import scala.concurrent.duration.{TimeUnit, Duration}
25
- import java.util.concurrent.TimeUnit.SECONDS
20
+ import s3.website.S3.S3Settings
21
+ import s3.website.model.Error.isClientError
26
22
 
27
23
  class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
28
24
 
29
- def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): S3PushResult =
25
+ def upload(upload: Upload with UploadTypeResolved)(implicit a: Attempt = 1, config: Config): Future[Either[FailedUpload, SuccessfulUpload]] =
30
26
  Future {
31
27
  s3Settings.s3Client(config) putObject toPutObjectRequest(upload)
32
28
  val report = SuccessfulUpload(upload)
33
29
  info(report)
34
30
  Right(report)
35
31
  } recoverWith retry(
36
- createReport = error => FailedUpload(upload.s3Key, error),
32
+ createFailureReport = error => FailedUpload(upload.s3Key, error),
37
33
  retryAction = newAttempt => this.upload(upload)(newAttempt, config)
38
34
  )
39
35
 
40
- def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): S3PushResult =
36
+ def delete(s3Key: String)(implicit a: Attempt = 1, config: Config): Future[Either[FailedDelete, SuccessfulDelete]] =
41
37
  Future {
42
38
  s3Settings.s3Client(config) deleteObject(config.s3_bucket, s3Key)
43
39
  val report = SuccessfulDelete(s3Key)
44
40
  info(report)
45
41
  Right(report)
46
42
  } recoverWith retry(
47
- createReport = error => FailedDelete(s3Key, error),
43
+ createFailureReport = error => FailedDelete(s3Key, error),
48
44
  retryAction = newAttempt => this.delete(s3Key)(newAttempt, config)
49
45
  )
50
46
 
51
- type S3PushResult = Future[Either[PushFailureReport, PushSuccessReport]]
52
-
53
- type Attempt = Int
54
-
55
- def retry(createReport: (Throwable) => PushFailureReport, retryAction: (Attempt) => S3PushResult)(implicit attempt: Attempt):
56
- PartialFunction[Throwable, S3PushResult] = {
57
- case error: Throwable =>
58
- val failureReport = createReport(error)
59
- if (attempt == 6) {
60
- info(failureReport)
61
- Future(Left(failureReport))
62
- } else {
63
- val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
64
- pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
65
- Thread.sleep(sleepDuration.toMillis)
66
- retryAction(attempt + 1)
67
- }
68
- }
69
-
70
47
  def toPutObjectRequest(upload: Upload)(implicit config: Config) =
71
48
  upload.essence.fold(
72
49
  redirect => {
@@ -99,36 +76,49 @@ class S3(implicit s3Settings: S3Settings, executor: ExecutionContextExecutor) {
99
76
  object S3 {
100
77
  def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
101
78
 
102
- def resolveS3Files(implicit config: Config, s3Settings: S3Settings): Either[Error, Stream[S3File]] = Try {
103
- objectSummaries()
104
- } match {
105
- case Success(remoteFiles) =>
106
- Right(remoteFiles)
107
- case Failure(error) if error.isInstanceOf[AmazonClientException] =>
108
- Left(UserError(error.getMessage))
109
- case Failure(error) =>
110
- Left(IOError(error))
111
- }
112
-
113
- def objectSummaries(nextMarker: Option[String] = None)(implicit config: Config, s3Settings: S3Settings): Stream[S3File] = {
79
+ def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil)
80
+ (implicit attempt: Attempt = 1, config: Config, s3Settings: S3Settings, ec: ExecutionContextExecutor): ObjectListingResult = Future {
81
+ nextMarker.foreach(m => info(s"Fetching the next part of the object listing from S3 (starting from $m)"))
114
82
  val objects: ObjectListing = s3Settings.s3Client(config).listObjects({
115
83
  val req = new ListObjectsRequest()
116
84
  req.setBucketName(config.s3_bucket)
117
85
  nextMarker.foreach(req.setMarker)
118
86
  req
119
87
  })
120
- val summaries = (objects.getObjectSummaries map (S3File(_))).toStream
121
- if (objects.isTruncated)
122
- summaries #::: objectSummaries(Some(objects.getNextMarker)) // Call the next Get Bucket request lazily
123
- else
124
- summaries
125
- }
88
+ objects
89
+ } flatMap { objects =>
90
+ val s3Files = alreadyResolved ++ (objects.getObjectSummaries.toIndexedSeq.toSeq map (S3File(_)))
91
+ Option(objects.getNextMarker)
92
+ .fold(Future(Right(s3Files)): ObjectListingResult)(nextMarker => resolveS3Files(Some(nextMarker), s3Files))
93
+ } recoverWith retry(
94
+ createFailureReport = error => UserError(s"Failed to fetch an object listing (${error.getMessage})"),
95
+ retryAction = nextAttempt => resolveS3Files(nextMarker, alreadyResolved)(nextAttempt, config, s3Settings, ec)
96
+ )
97
+
98
+ type ObjectListingResult = Future[Either[ErrorReport, Seq[S3File]]]
126
99
 
127
100
  sealed trait PushFailureReport extends FailureReport
128
101
  sealed trait PushSuccessReport extends SuccessReport {
129
102
  def s3Key: String
130
103
  }
131
104
 
105
+ def retry[L <: Report, R](createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
106
+ (implicit attempt: Attempt, s3Settings: S3Settings, ec: ExecutionContextExecutor):
107
+ PartialFunction[Throwable, Future[Either[L, R]]] = {
108
+ case error: Throwable if attempt == 6 || isClientError(error) =>
109
+ val failureReport = createFailureReport(error)
110
+ fail(failureReport.reportMessage)
111
+ Future(Left(failureReport))
112
+ case error: Throwable =>
113
+ val failureReport = createFailureReport(error)
114
+ val sleepDuration = Duration(fibs.drop(attempt + 1).head, s3Settings.retrySleepTimeUnit)
115
+ pending(s"${failureReport.reportMessage}. Trying again in $sleepDuration.")
116
+ Thread.sleep(sleepDuration.toMillis)
117
+ retryAction(attempt + 1)
118
+ }
119
+
120
+ type Attempt = Int
121
+
132
122
  case class SuccessfulUpload(upload: Upload with UploadTypeResolved) extends PushSuccessReport {
133
123
  def reportMessage =
134
124
  upload.uploadType match {
@@ -3,6 +3,7 @@ package s3.website.model
3
3
  import scala.util.{Failure, Try}
4
4
  import scala.collection.JavaConversions._
5
5
  import s3.website.Ruby.rubyRuntime
6
+ import s3.website.ErrorReport
6
7
 
7
8
  case class Config(
8
9
  s3_id: String,
@@ -22,7 +23,7 @@ case class Config(
22
23
  )
23
24
 
24
25
  object Config {
25
- def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Either[Boolean, Seq[String]]]] = {
26
+ def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {
26
27
  val yamlValue = for {
27
28
  optionalValue <- loadOptionalValue(key)
28
29
  } yield {
@@ -35,7 +36,7 @@ object Config {
35
36
  yamlValue getOrElse Left(UserError(s"The key $key has to have a boolean or [string] value"))
36
37
  }
37
38
 
38
- def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Either[String, Seq[String]]]] = {
39
+ def loadOptionalStringOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[String, Seq[String]]]] = {
39
40
  val yamlValue = for {
40
41
  valueOption <- loadOptionalValue(key)
41
42
  } yield {
@@ -48,7 +49,7 @@ object Config {
48
49
  yamlValue getOrElse Left(UserError(s"The key $key has to have a string or [string] value"))
49
50
  }
50
51
 
51
- def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Either[Int, Map[String, Int]]]] = {
52
+ def loadMaxAge(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Int, Map[String, Int]]]] = {
52
53
  val key = "max_age"
53
54
  val yamlValue = for {
54
55
  maxAgeOption <- loadOptionalValue(key)
@@ -62,7 +63,7 @@ object Config {
62
63
  yamlValue getOrElse Left(UserError(s"The key $key has to have an int or (string -> int) value"))
63
64
  }
64
65
 
65
- def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[S3Endpoint]] =
66
+ def loadEndpoint(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[S3Endpoint]] =
66
67
  loadOptionalString("s3_endpoint").right flatMap { endpointString =>
67
68
  endpointString.map(S3Endpoint.forString) match {
68
69
  case Some(Right(endpoint)) => Right(Some(endpoint))
@@ -71,17 +72,17 @@ object Config {
71
72
  }
72
73
  }
73
74
 
74
- def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Map[String, String]]] = {
75
+ def loadRedirects(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Map[String, String]]] = {
75
76
  val key = "redirects"
76
77
  val yamlValue = for {
77
78
  redirectsOption <- loadOptionalValue(key)
78
79
  redirects <- Try(redirectsOption.map(_.asInstanceOf[java.util.Map[String,String]].toMap))
79
80
  } yield Right(redirects)
80
81
 
81
- yamlValue getOrElse Left(UserError(s"The key $key has to have an int or (string -> int) value"))
82
+ yamlValue getOrElse Left(UserError(s"The key $key has to have a (string -> string) value"))
82
83
  }
83
84
 
84
- def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, String] = {
85
+ def loadRequiredString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, String] = {
85
86
  val yamlValue = for {
86
87
  valueOption <- loadOptionalValue(key)
87
88
  stringValue <- Try(valueOption.asInstanceOf[Option[String]].get)
@@ -94,7 +95,7 @@ object Config {
94
95
  }
95
96
  }
96
97
 
97
- def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[String]] = {
98
+ def loadOptionalString(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[String]] = {
98
99
  val yamlValueOption = for {
99
100
  valueOption <- loadOptionalValue(key)
100
101
  optionalString <- Try(valueOption.asInstanceOf[Option[String]])
@@ -107,7 +108,7 @@ object Config {
107
108
  }
108
109
  }
109
110
 
110
- def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Boolean]] = {
111
+ def loadOptionalBoolean(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Boolean]] = {
111
112
  val yamlValueOption = for {
112
113
  valueOption <- loadOptionalValue(key)
113
114
  optionalBoolean <- Try(valueOption.asInstanceOf[Option[Boolean]])
@@ -120,7 +121,7 @@ object Config {
120
121
  }
121
122
  }
122
123
 
123
- def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[Error, Option[Int]] = {
124
+ def loadOptionalInt(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Int]] = {
124
125
  val yamlValueOption = for {
125
126
  valueOption <- loadOptionalValue(key)
126
127
  optionalInt <- Try(valueOption.asInstanceOf[Option[Int]])
@@ -139,12 +140,16 @@ object Config {
139
140
  }
140
141
 
141
142
  def erbEval(erbString: String): Try[String] = Try {
142
- val erbStringWithoutComments = erbString.replaceAll("#.*", "")
143
+ val erbStringWithoutComments = erbString.replaceAll("^\\s*#.*", "")
143
144
  rubyRuntime.evalScriptlet(
144
145
  s"""
145
- require 'erb'
146
- ERB.new("$erbStringWithoutComments").result
147
- """
146
+ |require 'erb'
147
+ |
148
+ |str = <<-ERBSTR
149
+ |$erbStringWithoutComments
150
+ |ERBSTR
151
+ |ERB.new(str).result
152
+ """.stripMargin
148
153
  ).asJavaString()
149
154
  }
150
155
 
@@ -10,13 +10,14 @@ import s3.website.Logger._
10
10
  import scala.util.Failure
11
11
  import s3.website.model.Config.UnsafeYaml
12
12
  import scala.util.Success
13
+ import s3.website.ErrorReport
13
14
 
14
15
  case class Site(rootDirectory: String, config: Config) {
15
16
  def resolveS3Key(file: File) = file.getAbsolutePath.replace(rootDirectory, "").replaceFirst("^/", "")
16
17
  }
17
18
 
18
19
  object Site {
19
- def loadSite(yamlConfigPath: String, siteRootDirectory: String): Either[Error, Site] = {
20
+ def loadSite(yamlConfigPath: String, siteRootDirectory: String): Either[ErrorReport, Site] = {
20
21
  val yamlObjectTry = for {
21
22
  yamlString <- Try(fromFile(new File(yamlConfigPath)).mkString)
22
23
  yamlWithErbEvaluated <- erbEval(yamlString)
@@ -25,7 +26,7 @@ object Site {
25
26
  yamlObjectTry match {
26
27
  case Success(yamlObject) =>
27
28
  implicit val unsafeYaml = UnsafeYaml(yamlObject)
28
- val config: Either[Error, Config] = for {
29
+ val config: Either[ErrorReport, Config] = for {
29
30
  s3_id <- loadRequiredString("s3_id").right
30
31
  s3_secret <- loadRequiredString("s3_secret").right
31
32
  s3_bucket <- loadRequiredString("s3_bucket").right
@@ -1,11 +1,16 @@
1
1
  package s3.website.model
2
2
 
3
- trait Error {
4
- def message: String
3
+ import com.amazonaws.AmazonServiceException
4
+ import com.amazonaws.AmazonServiceException.ErrorType.Client
5
+ import s3.website.ErrorReport
6
+
7
+ object Error {
8
+ def isClientError(error: Throwable) =
9
+ error.isInstanceOf[AmazonServiceException] && error.asInstanceOf[AmazonServiceException].getErrorType == Client
5
10
  }
6
11
 
7
- case class UserError(message: String) extends Error
12
+ case class UserError(reportMessage: String) extends ErrorReport
8
13
 
9
- case class IOError(exception: Throwable) extends Error {
10
- def message = exception.getMessage
14
+ case class IOError(exception: Throwable) extends ErrorReport {
15
+ def reportMessage = exception.getMessage
11
16
  }
@@ -9,6 +9,7 @@ import java.util.zip.GZIPOutputStream
9
9
  import org.apache.commons.io.IOUtils
10
10
  import org.apache.tika.Tika
11
11
  import s3.website.Ruby._
12
+ import s3.website._
12
13
  import s3.website.model.Encoding.Gzip
13
14
  import scala.util.Failure
14
15
  import scala.Some
@@ -59,7 +60,7 @@ case class LocalFile(
59
60
  ) extends S3KeyProvider
60
61
 
61
62
  object LocalFile {
62
- def toUpload(localFile: LocalFile)(implicit config: Config): Either[Error, Upload] = Try {
63
+ def toUpload(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] = Try {
63
64
  def fis(file: File): InputStream = new FileInputStream(file)
64
65
  def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
65
66
  val sourceFile: File = localFile
@@ -107,7 +108,7 @@ object LocalFile {
107
108
 
108
109
  lazy val tika = new Tika()
109
110
 
110
- def resolveLocalFiles(implicit site: Site): Either[Error, Seq[LocalFile]] = Try {
111
+ def resolveLocalFiles(implicit site: Site): Either[ErrorReport, Seq[LocalFile]] = Try {
111
112
  val files = recursiveListFiles(new File(site.rootDirectory)).filterNot(_.isDirectory)
112
113
  files map { file =>
113
114
  val s3Key = site.resolveS3Key(file)
@@ -7,4 +7,6 @@ package object website {
7
7
  trait SuccessReport extends Report
8
8
 
9
9
  trait FailureReport extends Report
10
+
11
+ trait ErrorReport extends Report
10
12
  }
@@ -18,13 +18,15 @@ import s3.website.S3.{S3Settings, S3ClientProvider}
18
18
  import scala.collection.JavaConversions._
19
19
  import s3.website.model.NewFile
20
20
  import scala.Some
21
- import com.amazonaws.AmazonServiceException
21
+ import com.amazonaws.{AmazonClientException, AmazonServiceException}
22
22
  import org.apache.commons.codec.digest.DigestUtils.md5Hex
23
23
  import s3.website.CloudFront.{CloudFrontSettings, CloudFrontClientProvider}
24
24
  import com.amazonaws.services.cloudfront.AmazonCloudFront
25
25
  import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
26
26
  import org.mockito.stubbing.Answer
27
27
  import org.mockito.invocation.InvocationOnMock
28
+ import com.amazonaws.AmazonServiceException.ErrorType
29
+ import java.util.concurrent.atomic.AtomicInteger
28
30
 
29
31
  class S3WebsiteSpec extends Specification {
30
32
 
@@ -101,6 +103,17 @@ class S3WebsiteSpec extends Specification {
101
103
  verify(amazonS3Client, times(6)).putObject(Matchers.any(classOf[PutObjectRequest]))
102
104
  }
103
105
 
106
+ "not try again if the upload fails on because of the client" in new SiteDirectory with MockAWS {
107
+ implicit val site = siteWithFilesAndContent(localFilesWithContent = ("index.html", "<h1>hello</h1>") :: Nil)
108
+ when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow {
109
+ val e = new AmazonServiceException("your credentials are incorrect")
110
+ e.setErrorType(ErrorType.Client)
111
+ e
112
+ }
113
+ Push.pushSite
114
+ verify(amazonS3Client, times(1)).putObject(Matchers.any(classOf[PutObjectRequest]))
115
+ }
116
+
104
117
  "try again if the delete fails" in new SiteDirectory with MockAWS {
105
118
  implicit val site = buildSite()
106
119
  setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
@@ -108,6 +121,14 @@ class S3WebsiteSpec extends Specification {
108
121
  Push.pushSite
109
122
  verify(amazonS3Client, times(6)).deleteObject(Matchers.anyString(), Matchers.anyString())
110
123
  }
124
+
125
+ "try again if the object listing fails" in new SiteDirectory with MockAWS {
126
+ implicit val site = buildSite()
127
+ setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
128
+ objectListingFailsAndThenSucceeds(howManyFailures = 5)
129
+ Push.pushSite
130
+ verify(amazonS3Client, times(6)).listObjects(Matchers.any(classOf[ListObjectsRequest]))
131
+ }
111
132
  }
112
133
 
113
134
  "push with CloudFront" should {
@@ -184,6 +205,12 @@ class S3WebsiteSpec extends Specification {
184
205
  Push.pushSite must equalTo(1)
185
206
  }
186
207
 
208
+ "be 1 if any of the redirects fails" in new SiteDirectory with MockAWS {
209
+ implicit val site = buildSite(defaultConfig.copy(redirects = Some(Map("index.php" -> "/index.html"))))
210
+ when(amazonS3Client.putObject(Matchers.any(classOf[PutObjectRequest]))).thenThrow(new AmazonServiceException("AWS failed"))
211
+ Push.pushSite must equalTo(1)
212
+ }
213
+
187
214
  "be 0 if CloudFront invalidations and uploads succeed"in new SiteDirectory with MockAWS {
188
215
  implicit val site = siteWithFiles(
189
216
  config = defaultConfig.copy(cloudfront_distribution_id = Some("EGM1J2JJX9Z")),
@@ -212,6 +239,13 @@ class S3WebsiteSpec extends Specification {
212
239
  uploadFailsAndThenSucceeds(howManyFailures = 6)
213
240
  Push.pushSite must equalTo(1)
214
241
  }
242
+
243
+ "be 1 if an object listing fails" in new SiteDirectory with MockAWS {
244
+ implicit val site = buildSite()
245
+ setS3Files(S3File("old.html", md5Hex("<h1>old text</h1>")))
246
+ objectListingFailsAndThenSucceeds(howManyFailures = 6)
247
+ Push.pushSite must equalTo(1)
248
+ }
215
249
  }
216
250
 
217
251
  "s3_website.yml file" should {
@@ -408,30 +442,32 @@ class S3WebsiteSpec extends Specification {
408
442
 
409
443
  val s3 = new S3()
410
444
 
411
- def uploadFailsAndThenSucceeds(howManyFailures: Int) {
412
- var callCount = 0
413
- doAnswer(new Answer[PutObjectResult] {
414
- override def answer(invocation: InvocationOnMock) = {
415
- callCount += 1
416
- if (callCount <= howManyFailures)
417
- throw new AmazonServiceException("AWS is temporarily down")
418
- else
419
- mock(classOf[PutObjectResult])
420
- }
421
- }).when(amazonS3Client).putObject(Matchers.anyObject())
445
+ def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
446
+ doAnswer(temporaryFailure(classOf[PutObjectResult]))
447
+ .when(amazonS3Client)
448
+ .putObject(Matchers.anyObject())
422
449
  }
423
450
 
424
- def deleteFailsAndThenSucceeds(howManyFailures: Int) {
425
- var callCount = 0
426
- doAnswer(new Answer[DeleteObjectRequest] {
427
- override def answer(invocation: InvocationOnMock) = {
428
- callCount += 1
429
- if (callCount <= howManyFailures)
430
- throw new AmazonServiceException("AWS is temporarily down")
431
- else
432
- mock(classOf[DeleteObjectRequest])
433
- }
434
- }).when(amazonS3Client).deleteObject(Matchers.anyString(), Matchers.anyString())
451
+ def deleteFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
452
+ doAnswer(temporaryFailure(classOf[DeleteObjectRequest]))
453
+ .when(amazonS3Client)
454
+ .deleteObject(Matchers.anyString(), Matchers.anyString())
455
+ }
456
+
457
+ def objectListingFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
458
+ doAnswer(temporaryFailure(classOf[ObjectListing]))
459
+ .when(amazonS3Client)
460
+ .listObjects(Matchers.any(classOf[ListObjectsRequest]))
461
+ }
462
+
463
+ def temporaryFailure[T](clazz: Class[T])(implicit callCount: AtomicInteger, howManyFailures: Int) = new Answer[T] {
464
+ def answer(invocation: InvocationOnMock) = {
465
+ callCount.incrementAndGet()
466
+ if (callCount.get() <= howManyFailures)
467
+ throw new AmazonServiceException("AWS is temporarily down")
468
+ else
469
+ mock(clazz)
470
+ }
435
471
  }
436
472
 
437
473
  def asSeenByS3Client(upload: Upload)(implicit config: Config): PutObjectRequest = {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3_website_monadic
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.6
4
+ version: 0.0.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Lauri Lehmijoki
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-05-07 00:00:00.000000000 Z
11
+ date: 2014-05-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: aws-sdk