s3_website_monadic 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/s3_website.gemspec +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +17 -6
- data/src/main/scala/s3/website/Push.scala +20 -6
- data/src/main/scala/s3/website/S3.scala +15 -9
- data/src/main/scala/s3/website/Utils.scala +65 -0
- data/src/main/scala/s3/website/model/Site.scala +7 -3
- data/src/main/scala/s3/website/package.scala +10 -0
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 001860d8f4b29eae3c6d2f4e71d4e815a09bd068
|
4
|
+
data.tar.gz: 2c39928790f622d404855dc548858bd06960c46a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1c33fa0f3299e3852a67bc2267f9afc8b6895833f9ce5007f5615aca50080f443fbe4f1ef84b92a36d062790179e3819686f1e797b4c8851b9e8340f5dd6d0d1
|
7
|
+
data.tar.gz: 620e2e04c5d145b2e86be58662c5669223f446ab480d8dd5259209c8b6ed2c0ea476d058160a0c545bee0677c766b608dfe5f997033c06e85446af2c0a99de0a
|
data/s3_website.gemspec
CHANGED
@@ -9,6 +9,12 @@ import scala.collection.JavaConversions._
|
|
9
9
|
import scala.concurrent.duration._
|
10
10
|
import s3.website.S3.{SuccessfulUpload, PushSuccessReport}
|
11
11
|
import com.amazonaws.auth.BasicAWSCredentials
|
12
|
+
import s3.website.Logger._
|
13
|
+
import s3.website.S3.SuccessfulUpload
|
14
|
+
import scala.util.Failure
|
15
|
+
import s3.website.CloudFront.SuccessfulInvalidation
|
16
|
+
import scala.util.Success
|
17
|
+
import s3.website.CloudFront.FailedInvalidation
|
12
18
|
|
13
19
|
class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUnit) {
|
14
20
|
|
@@ -18,7 +24,7 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
|
|
18
24
|
val invalidationReq = new CreateInvalidationRequest(distributionId, invalidationBatch)
|
19
25
|
cfClient(config).createInvalidation(invalidationReq)
|
20
26
|
val result = SuccessfulInvalidation(invalidationBatch.getPaths.getItems.size())
|
21
|
-
|
27
|
+
info(result)
|
22
28
|
result
|
23
29
|
} recoverWith {
|
24
30
|
case e: TooManyInvalidationsInProgressException =>
|
@@ -26,7 +32,7 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
|
|
26
32
|
(fibs drop attempt).head min 15, /* AWS docs way that invalidations complete in 15 minutes */
|
27
33
|
sleepUnit
|
28
34
|
)
|
29
|
-
|
35
|
+
pending(maxInvalidationsExceededInfo)
|
30
36
|
Thread.sleep(duration.toMillis)
|
31
37
|
tryInvalidate(attempt + 1)
|
32
38
|
}
|
@@ -35,8 +41,9 @@ class CloudFront(implicit cfClient: CloudFrontClientProvider, sleepUnit: TimeUni
|
|
35
41
|
case Success(res) =>
|
36
42
|
Right(res)
|
37
43
|
case Failure(err) =>
|
38
|
-
|
39
|
-
|
44
|
+
val report = FailedInvalidation(err)
|
45
|
+
info(report)
|
46
|
+
Left(report)
|
40
47
|
}
|
41
48
|
}
|
42
49
|
|
@@ -61,9 +68,13 @@ object CloudFront {
|
|
61
68
|
|
62
69
|
type CloudFrontClientProvider = (Config) => AmazonCloudFront
|
63
70
|
|
64
|
-
case class SuccessfulInvalidation(invalidatedItemsCount: Int)
|
71
|
+
case class SuccessfulInvalidation(invalidatedItemsCount: Int) extends SuccessReport {
|
72
|
+
def reportMessage = s"Invalidated $invalidatedItemsCount item(s) on CloudFront"
|
73
|
+
}
|
65
74
|
|
66
|
-
case class FailedInvalidation()
|
75
|
+
case class FailedInvalidation(error: Throwable) extends FailureReport{
|
76
|
+
def reportMessage = s"Failed to invalidate the CloudFront distribution (${error.getMessage})"
|
77
|
+
}
|
67
78
|
|
68
79
|
def awsCloudFrontClient(config: Config) =
|
69
80
|
new AmazonCloudFrontClient(new BasicAWSCredentials(config.s3_id, config.s3_secret))
|
@@ -23,6 +23,15 @@ import s3.website.S3.SuccessfulDelete
|
|
23
23
|
import s3.website.CloudFront.SuccessfulInvalidation
|
24
24
|
import s3.website.S3.SuccessfulUpload
|
25
25
|
import s3.website.CloudFront.FailedInvalidation
|
26
|
+
import s3.website.Logger._
|
27
|
+
import s3.website.S3.SuccessfulDelete
|
28
|
+
import s3.website.CloudFront.SuccessfulInvalidation
|
29
|
+
import s3.website.S3.SuccessfulUpload
|
30
|
+
import s3.website.CloudFront.FailedInvalidation
|
31
|
+
import s3.website.S3.SuccessfulDelete
|
32
|
+
import s3.website.CloudFront.SuccessfulInvalidation
|
33
|
+
import s3.website.S3.SuccessfulUpload
|
34
|
+
import s3.website.CloudFront.FailedInvalidation
|
26
35
|
|
27
36
|
object Push {
|
28
37
|
|
@@ -33,7 +42,7 @@ object Push {
|
|
33
42
|
cloudFrontClientProvider: CloudFrontClientProvider = CloudFront.awsCloudFrontClient,
|
34
43
|
cloudFrontSleepTimeUnit: TimeUnit = MINUTES
|
35
44
|
): ExitCode = {
|
36
|
-
|
45
|
+
info(s"Deploying ${site.rootDirectory}/* to ${site.config.s3_bucket}")
|
37
46
|
val utils: Utils = new Utils
|
38
47
|
|
39
48
|
val errorsOrReports = for {
|
@@ -93,11 +102,10 @@ object Push {
|
|
93
102
|
def afterPushFinished(errorsOrFinishedUploads: Either[Error, FinishedPushOperations], invalidationSucceeded: Option[Boolean])(implicit config: Config): ExitCode = {
|
94
103
|
errorsOrFinishedUploads.right.foreach { finishedUploads =>
|
95
104
|
val pushCounts = pushCountsToString(resolvePushCounts(finishedUploads))
|
96
|
-
|
97
|
-
println(s"Go visit: http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
105
|
+
info(s"Summary: $pushCounts")
|
98
106
|
}
|
99
|
-
errorsOrFinishedUploads.left foreach (err =>
|
100
|
-
errorsOrFinishedUploads.fold(
|
107
|
+
errorsOrFinishedUploads.left foreach (err => fail(s"Encountered error: ${err.message}"))
|
108
|
+
val exitCode = errorsOrFinishedUploads.fold(
|
101
109
|
_ => 1,
|
102
110
|
finishedUploads => finishedUploads.foldLeft(0) { (memo, finishedUpload) =>
|
103
111
|
memo + finishedUpload.fold(
|
@@ -109,6 +117,12 @@ object Push {
|
|
109
117
|
) max invalidationSucceeded.fold(0)(allInvalidationsSucceeded =>
|
110
118
|
if (allInvalidationsSucceeded) 0 else 1
|
111
119
|
)
|
120
|
+
|
121
|
+
if (exitCode == 0)
|
122
|
+
info(s"Successfully pushed the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
123
|
+
else
|
124
|
+
fail(s"Failed to push the website to http://${config.s3_bucket}.${config.s3_endpoint.s3WebsiteHostname}")
|
125
|
+
exitCode
|
112
126
|
}
|
113
127
|
|
114
128
|
def awaitForUploads(uploadReports: PushReports)(implicit executor: ExecutionContextExecutor): FinishedPushOperations =
|
@@ -180,7 +194,7 @@ object Push {
|
|
180
194
|
threadPool.shutdownNow()
|
181
195
|
pushStatus
|
182
196
|
}
|
183
|
-
errorOrPushStatus.left foreach (err =>
|
197
|
+
errorOrPushStatus.left foreach (err => fail(s"Could not load the site: ${err.message}"))
|
184
198
|
System.exit(errorOrPushStatus.fold(_ => 1, pushStatus => pushStatus))
|
185
199
|
}
|
186
200
|
}
|
@@ -17,6 +17,16 @@ import scala.Some
|
|
17
17
|
import s3.website.model.IOError
|
18
18
|
import scala.util.Success
|
19
19
|
import s3.website.model.UserError
|
20
|
+
import s3.website.Logger._
|
21
|
+
import s3.website.S3.SuccessfulUpload
|
22
|
+
import s3.website.S3.SuccessfulDelete
|
23
|
+
import s3.website.S3.FailedUpload
|
24
|
+
import scala.util.Failure
|
25
|
+
import scala.Some
|
26
|
+
import s3.website.S3.FailedDelete
|
27
|
+
import s3.website.model.IOError
|
28
|
+
import scala.util.Success
|
29
|
+
import s3.website.model.UserError
|
20
30
|
|
21
31
|
class S3(implicit s3Client: S3ClientProvider) {
|
22
32
|
|
@@ -24,7 +34,7 @@ class S3(implicit s3Client: S3ClientProvider) {
|
|
24
34
|
Future {
|
25
35
|
s3Client(config) putObject toPutObjectRequest(upload)
|
26
36
|
val report = SuccessfulUpload(upload)
|
27
|
-
|
37
|
+
info(report)
|
28
38
|
Right(report)
|
29
39
|
} recover usingErrorHandler { error =>
|
30
40
|
FailedUpload(upload.s3Key, error)
|
@@ -34,7 +44,7 @@ class S3(implicit s3Client: S3ClientProvider) {
|
|
34
44
|
Future {
|
35
45
|
s3Client(config) deleteObject(config.s3_bucket, s3Key)
|
36
46
|
val report = SuccessfulDelete(s3Key)
|
37
|
-
|
47
|
+
info(report)
|
38
48
|
Right(report)
|
39
49
|
} recover usingErrorHandler { error =>
|
40
50
|
FailedDelete(s3Key, error)
|
@@ -43,7 +53,7 @@ class S3(implicit s3Client: S3ClientProvider) {
|
|
43
53
|
def usingErrorHandler[T <: PushFailureReport, F <: PushFailureReport](f: (Throwable) => T): PartialFunction[Throwable, Either[T, F]] = {
|
44
54
|
case error =>
|
45
55
|
val report = f(error)
|
46
|
-
|
56
|
+
info(report)
|
47
57
|
Left(report)
|
48
58
|
}
|
49
59
|
|
@@ -104,12 +114,8 @@ object S3 {
|
|
104
114
|
summaries
|
105
115
|
}
|
106
116
|
|
107
|
-
sealed trait
|
108
|
-
|
109
|
-
}
|
110
|
-
|
111
|
-
sealed trait PushFailureReport extends PushItemReport
|
112
|
-
sealed trait PushSuccessReport extends PushItemReport {
|
117
|
+
sealed trait PushFailureReport extends FailureReport
|
118
|
+
sealed trait PushSuccessReport extends SuccessReport {
|
113
119
|
def s3Key: String
|
114
120
|
}
|
115
121
|
|
@@ -11,3 +11,68 @@ class Utils(implicit config: Config) {
|
|
11
11
|
parallelSeq
|
12
12
|
}
|
13
13
|
}
|
14
|
+
|
15
|
+
object Logger {
|
16
|
+
import Rainbow._
|
17
|
+
def info(msg: String) = println(s"[${"info".blue}] $msg")
|
18
|
+
def fail(msg: String) = println(s"[${"fail".red}] $msg")
|
19
|
+
|
20
|
+
def info(report: SuccessReport) = println(s"[${"succ".green}] ${report.reportMessage}")
|
21
|
+
def info(report: FailureReport) = fail(report.reportMessage)
|
22
|
+
|
23
|
+
def pending(msg: String) = println(s"[${"wait".yellow}] $msg")
|
24
|
+
}
|
25
|
+
|
26
|
+
/**
|
27
|
+
* Idea copied from https://github.com/ktoso/scala-rainbow.
|
28
|
+
*/
|
29
|
+
object Rainbow {
|
30
|
+
implicit class RainbowString(val s: String) extends AnyVal {
|
31
|
+
import Console._
|
32
|
+
|
33
|
+
/** Colorize the given string foreground to ANSI black */
|
34
|
+
def black = BLACK + s + RESET
|
35
|
+
/** Colorize the given string foreground to ANSI red */
|
36
|
+
def red = RED + s + RESET
|
37
|
+
/** Colorize the given string foreground to ANSI red */
|
38
|
+
def green = GREEN + s + RESET
|
39
|
+
/** Colorize the given string foreground to ANSI red */
|
40
|
+
def yellow = YELLOW + s + RESET
|
41
|
+
/** Colorize the given string foreground to ANSI red */
|
42
|
+
def blue = BLUE + s + RESET
|
43
|
+
/** Colorize the given string foreground to ANSI red */
|
44
|
+
def magenta = MAGENTA + s + RESET
|
45
|
+
/** Colorize the given string foreground to ANSI red */
|
46
|
+
def cyan = CYAN + s + RESET
|
47
|
+
/** Colorize the given string foreground to ANSI red */
|
48
|
+
def white = WHITE + s + RESET
|
49
|
+
|
50
|
+
/** Colorize the given string background to ANSI red */
|
51
|
+
def onBlack = BLACK_B + s + RESET
|
52
|
+
/** Colorize the given string background to ANSI red */
|
53
|
+
def onRed = RED_B+ s + RESET
|
54
|
+
/** Colorize the given string background to ANSI red */
|
55
|
+
def onGreen = GREEN_B+ s + RESET
|
56
|
+
/** Colorize the given string background to ANSI red */
|
57
|
+
def onYellow = YELLOW_B + s + RESET
|
58
|
+
/** Colorize the given string background to ANSI red */
|
59
|
+
def onBlue = BLUE_B+ s + RESET
|
60
|
+
/** Colorize the given string background to ANSI red */
|
61
|
+
def onMagenta = MAGENTA_B + s + RESET
|
62
|
+
/** Colorize the given string background to ANSI red */
|
63
|
+
def onCyan = CYAN_B+ s + RESET
|
64
|
+
/** Colorize the given string background to ANSI red */
|
65
|
+
def onWhite = WHITE_B+ s + RESET
|
66
|
+
|
67
|
+
/** Make the given string bold */
|
68
|
+
def bold = BOLD + s + RESET
|
69
|
+
/** Underline the given string */
|
70
|
+
def underlined = UNDERLINED + s + RESET
|
71
|
+
/** Make the given string blink (some terminals may turn this off) */
|
72
|
+
def blink = BLINK + s + RESET
|
73
|
+
/** Reverse the ANSI colors of the given string */
|
74
|
+
def reversed = REVERSED + s + RESET
|
75
|
+
/** Make the given string invisible using ANSI color codes */
|
76
|
+
def invisible = INVISIBLE + s + RESET
|
77
|
+
}
|
78
|
+
}
|
@@ -1,11 +1,15 @@
|
|
1
1
|
package s3.website.model
|
2
2
|
|
3
3
|
import java.io.File
|
4
|
-
import scala.util.
|
4
|
+
import scala.util.Try
|
5
5
|
import org.yaml.snakeyaml.Yaml
|
6
6
|
import s3.website.model.Config._
|
7
7
|
import scala.io.Source.fromFile
|
8
8
|
import scala.language.postfixOps
|
9
|
+
import s3.website.Logger._
|
10
|
+
import scala.util.Failure
|
11
|
+
import s3.website.model.Config.UnsafeYaml
|
12
|
+
import scala.util.Success
|
9
13
|
|
10
14
|
case class Site(rootDirectory: String, config: Config) {
|
11
15
|
def resolveS3Key(file: File) = file.getAbsolutePath.replace(rootDirectory, "").replaceFirst("^/", "")
|
@@ -38,8 +42,8 @@ object Site {
|
|
38
42
|
concurrency_level <- loadOptionalInt("concurrency_level").right
|
39
43
|
redirects <- loadRedirects.right
|
40
44
|
} yield {
|
41
|
-
gzip_zopfli.foreach(_ =>
|
42
|
-
extensionless_mime_type.foreach(_ =>
|
45
|
+
gzip_zopfli.foreach(_ => info("zopfli is not currently supported"))
|
46
|
+
extensionless_mime_type.foreach(_ => info(
|
43
47
|
s"Ignoring the extensionless_mime_type setting in $yamlConfigPath. Counting on Apache Tika to infer correct mime types.")
|
44
48
|
)
|
45
49
|
Config(
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_website_monadic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lauri Lehmijoki
|
@@ -378,6 +378,7 @@ files:
|
|
378
378
|
- src/main/scala/s3/website/model/Site.scala
|
379
379
|
- src/main/scala/s3/website/model/errors.scala
|
380
380
|
- src/main/scala/s3/website/model/push.scala
|
381
|
+
- src/main/scala/s3/website/package.scala
|
381
382
|
- src/test/scala/s3/website/S3WebsiteSpec.scala
|
382
383
|
homepage: https://github.com/laurilehmijoki/s3_website
|
383
384
|
licenses:
|