s3_website_monadic 0.0.31 → 0.0.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/changelog.md +2 -0
- data/lib/s3_website/version.rb +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +8 -15
- data/src/main/scala/s3/website/Diff.scala +264 -25
- data/src/main/scala/s3/website/Logger.scala +61 -0
- data/src/main/scala/s3/website/Push.scala +69 -77
- data/src/main/scala/s3/website/S3.scala +81 -76
- data/src/main/scala/s3/website/model/Config.scala +8 -8
- data/src/main/scala/s3/website/model/S3Endpoint.scala +4 -2
- data/src/main/scala/s3/website/model/Site.scala +6 -3
- data/src/main/scala/s3/website/model/push.scala +72 -140
- data/src/main/scala/s3/website/model/ssg.scala +1 -1
- data/src/main/scala/s3/website/package.scala +23 -1
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +64 -25
- metadata +3 -4
- data/src/main/scala/s3/website/Utils.scala +0 -108
- data/src/main/scala/s3/website/model/errors.scala +0 -9
@@ -2,19 +2,15 @@ package s3.website.model
|
|
2
2
|
|
3
3
|
import com.amazonaws.services.s3.model.S3ObjectSummary
|
4
4
|
import java.io._
|
5
|
-
import scala.util.Try
|
6
|
-
import s3.website.model.Encoding._
|
7
5
|
import org.apache.commons.codec.digest.DigestUtils
|
8
6
|
import java.util.zip.GZIPOutputStream
|
9
|
-
import org.apache.commons.io.IOUtils
|
10
7
|
import org.apache.tika.Tika
|
11
8
|
import s3.website.Ruby._
|
12
9
|
import s3.website._
|
13
|
-
import s3.website.model.
|
14
|
-
import
|
15
|
-
import
|
16
|
-
import
|
17
|
-
import s3.website.model.Encoding.Zopfli
|
10
|
+
import s3.website.model.LocalFileFromDisk.tika
|
11
|
+
import s3.website.model.Encoding.encodingOnS3
|
12
|
+
import java.io.File.createTempFile
|
13
|
+
import org.apache.commons.io.IOUtils.copy
|
18
14
|
|
19
15
|
object Encoding {
|
20
16
|
|
@@ -23,104 +19,52 @@ object Encoding {
|
|
23
19
|
case class Gzip()
|
24
20
|
case class Zopfli()
|
25
21
|
|
26
|
-
def encodingOnS3(
|
27
|
-
|
22
|
+
def encodingOnS3(s3Key: String)(implicit config: Config): Option[Either[Gzip, Zopfli]] =
|
23
|
+
config.gzip.flatMap { (gzipSetting: Either[Boolean, Seq[String]]) =>
|
28
24
|
val shouldZipThisFile = gzipSetting.fold(
|
29
|
-
shouldGzip => defaultGzipExtensions exists
|
30
|
-
fileExtensions => fileExtensions exists
|
25
|
+
shouldGzip => defaultGzipExtensions exists s3Key.endsWith,
|
26
|
+
fileExtensions => fileExtensions exists s3Key.endsWith
|
31
27
|
)
|
32
|
-
if (shouldZipThisFile &&
|
28
|
+
if (shouldZipThisFile && config.gzip_zopfli.isDefined)
|
33
29
|
Some(Right(Zopfli()))
|
34
30
|
else if (shouldZipThisFile)
|
35
31
|
Some(Left(Gzip()))
|
36
32
|
else
|
37
33
|
None
|
38
34
|
}
|
39
|
-
|
40
|
-
type MD5 = String
|
41
|
-
}
|
42
|
-
|
43
|
-
sealed trait S3KeyProvider {
|
44
|
-
def s3Key: String
|
45
|
-
}
|
46
|
-
|
47
|
-
trait UploadTypeResolved {
|
48
|
-
def uploadType: UploadType
|
49
35
|
}
|
50
36
|
|
51
37
|
sealed trait UploadType // Sealed, so that we can avoid inexhaustive pattern matches more easily
|
52
38
|
|
53
39
|
case object NewFile extends UploadType
|
54
|
-
case object
|
40
|
+
case object FileUpdate extends UploadType
|
41
|
+
case object RedirectFile extends UploadType
|
42
|
+
|
43
|
+
case class LocalFileFromDisk(originalFile: File, uploadType: UploadType)(implicit site: Site) {
|
44
|
+
lazy val s3Key = site.resolveS3Key(originalFile)
|
55
45
|
|
56
|
-
|
57
|
-
s3Key: String,
|
58
|
-
originalFile: File,
|
59
|
-
encodingOnS3: Option[Either[Gzip, Zopfli]]
|
60
|
-
) extends S3KeyProvider {
|
46
|
+
lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
|
61
47
|
|
62
|
-
|
63
|
-
lazy val length = uploadFile.length()
|
48
|
+
lazy val lastModified = originalFile.lastModified
|
64
49
|
|
65
50
|
/**
|
66
51
|
* This is the file we should upload, because it contains the potentially gzipped contents of the original file.
|
67
52
|
*
|
68
53
|
* May throw an exception, so remember to call this in a Try or Future monad
|
69
54
|
*/
|
70
|
-
lazy val uploadFile: File =
|
71
|
-
.fold(originalFile)(algorithm => {
|
72
|
-
val tempFile = File.createTempFile(originalFile.getName, "gzip")
|
73
|
-
tempFile.deleteOnExit()
|
74
|
-
using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
|
75
|
-
IOUtils.copy(fis(originalFile), stream)
|
76
|
-
}
|
77
|
-
tempFile
|
78
|
-
})
|
79
|
-
|
80
|
-
/**
|
81
|
-
* May throw an exception, so remember to call this in a Try or Future monad
|
82
|
-
*/
|
83
|
-
lazy val md5 = using(fis(uploadFile)) { inputStream =>
|
84
|
-
DigestUtils.md5Hex(inputStream)
|
85
|
-
}
|
55
|
+
lazy val uploadFile: File = LocalFileFromDisk uploadFile originalFile
|
86
56
|
|
87
|
-
|
88
|
-
|
89
|
-
}
|
90
|
-
|
91
|
-
object LocalFile {
|
92
|
-
def toUpload(localFile: LocalFile)(implicit config: Config): Either[ErrorReport, Upload] = Try {
|
93
|
-
Upload(
|
94
|
-
s3Key = localFile.s3Key,
|
95
|
-
essence = Right(
|
96
|
-
UploadBody(
|
97
|
-
md5 = localFile.md5,
|
98
|
-
contentEncoding = localFile.encodingOnS3.map(_ => "gzip"),
|
99
|
-
contentLength = localFile.length,
|
100
|
-
maxAge = resolveMaxAge(localFile),
|
101
|
-
contentType = resolveContentType(localFile.originalFile),
|
102
|
-
openInputStream = () => new FileInputStream(localFile.uploadFile)
|
103
|
-
)
|
104
|
-
)
|
105
|
-
)
|
106
|
-
} match {
|
107
|
-
case Success(upload) => Right(upload)
|
108
|
-
case Failure(error) => Left(IOError(error))
|
109
|
-
}
|
110
|
-
|
111
|
-
lazy val tika = new Tika()
|
112
|
-
|
113
|
-
def resolveContentType(file: File) = {
|
114
|
-
val mimeType = tika.detect(file)
|
57
|
+
lazy val contentType = {
|
58
|
+
val mimeType = tika.detect(originalFile)
|
115
59
|
if (mimeType.startsWith("text/") || mimeType == "application/json")
|
116
60
|
mimeType + "; charset=utf-8"
|
117
61
|
else
|
118
62
|
mimeType
|
119
63
|
}
|
120
64
|
|
121
|
-
|
65
|
+
lazy val maxAge: Option[Int] = {
|
122
66
|
type GlobsMap = Map[String, Int]
|
123
|
-
config.max_age.flatMap { (intOrGlobs: Either[Int, GlobsMap]) =>
|
67
|
+
site.config.max_age.flatMap { (intOrGlobs: Either[Int, GlobsMap]) =>
|
124
68
|
type GlobsSeq = Seq[(String, Int)]
|
125
69
|
def respectMostSpecific(globs: GlobsMap): GlobsSeq = globs.toSeq.sortBy(_._1.length).reverse
|
126
70
|
intOrGlobs
|
@@ -129,7 +73,7 @@ object LocalFile {
|
|
129
73
|
(seconds: Int) => Some(seconds),
|
130
74
|
(globs: GlobsSeq) =>
|
131
75
|
globs.find { globAndInt =>
|
132
|
-
(rubyRuntime evalScriptlet s"File.fnmatch('${globAndInt._1}', '$
|
76
|
+
(rubyRuntime evalScriptlet s"File.fnmatch('${globAndInt._1}', '$s3Key')")
|
133
77
|
.toJava(classOf[Boolean])
|
134
78
|
.asInstanceOf[Boolean]
|
135
79
|
} map (_._2)
|
@@ -137,83 +81,71 @@ object LocalFile {
|
|
137
81
|
}
|
138
82
|
}
|
139
83
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
)
|
161
|
-
|
162
|
-
|
163
|
-
|
84
|
+
/**
|
85
|
+
* May throw an exception, so remember to call this in a Try or Future monad
|
86
|
+
*/
|
87
|
+
lazy val md5 = LocalFileFromDisk md5 originalFile
|
88
|
+
}
|
89
|
+
|
90
|
+
object LocalFileFromDisk {
|
91
|
+
lazy val tika = new Tika()
|
92
|
+
|
93
|
+
def md5(originalFile: File)(implicit site: Site) = using(fis { uploadFile(originalFile) }) { DigestUtils.md5Hex }
|
94
|
+
|
95
|
+
def uploadFile(originalFile: File)(implicit site: Site): File =
|
96
|
+
encodingOnS3(site resolveS3Key originalFile)
|
97
|
+
.fold(originalFile)(algorithm => {
|
98
|
+
val tempFile = createTempFile(originalFile.getName, "gzip")
|
99
|
+
tempFile.deleteOnExit()
|
100
|
+
using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
|
101
|
+
copy(fis(originalFile), stream)
|
102
|
+
}
|
103
|
+
tempFile
|
104
|
+
})
|
105
|
+
|
106
|
+
private[this] def fis(file: File): InputStream = new FileInputStream(file)
|
107
|
+
private[this] def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
|
108
|
+
}
|
164
109
|
|
110
|
+
object Files {
|
165
111
|
def recursiveListFiles(f: File): Seq[File] = {
|
166
112
|
val these = f.listFiles
|
167
113
|
these ++ these.filter(_.isDirectory).flatMap(recursiveListFiles)
|
168
114
|
}
|
169
|
-
}
|
170
|
-
|
171
|
-
case class Redirect(key: String, redirectTarget: String)
|
172
115
|
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
(
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
116
|
+
def listSiteFiles(implicit site: Site, logger: Logger) = {
|
117
|
+
def excludeFromUpload(s3Key: String) = {
|
118
|
+
val excludeByConfig = site.config.exclude_from_upload exists {
|
119
|
+
_.fold(
|
120
|
+
// For backward compatibility, use Ruby regex matching
|
121
|
+
(exclusionRegex: String) => rubyRegexMatches(s3Key, exclusionRegex),
|
122
|
+
(exclusionRegexes: Seq[String]) => exclusionRegexes exists (rubyRegexMatches(s3Key, _))
|
123
|
+
)
|
124
|
+
}
|
125
|
+
val doNotUpload = excludeByConfig || s3Key == "s3_website.yml"
|
126
|
+
if (doNotUpload) logger.debug(s"Excluded $s3Key from upload")
|
127
|
+
doNotUpload
|
184
128
|
}
|
129
|
+
recursiveListFiles(new File(site.rootDirectory))
|
130
|
+
.filterNot(_.isDirectory)
|
131
|
+
.filterNot(f => excludeFromUpload(site.resolveS3Key(f)))
|
185
132
|
}
|
186
133
|
}
|
187
134
|
|
188
|
-
case class
|
189
|
-
|
190
|
-
essence: Either[Redirect, UploadBody]
|
191
|
-
) extends S3KeyProvider {
|
192
|
-
|
193
|
-
def withUploadType(ut: UploadType) =
|
194
|
-
new Upload(s3Key, essence) with UploadTypeResolved {
|
195
|
-
def uploadType = ut
|
196
|
-
}
|
135
|
+
case class Redirect(s3Key: String, redirectTarget: String) {
|
136
|
+
def uploadType = RedirectFile
|
197
137
|
}
|
198
138
|
|
199
|
-
object
|
200
|
-
def
|
201
|
-
|
139
|
+
object Redirect {
|
140
|
+
def resolveRedirects(implicit config: Config): Seq[Redirect] =
|
141
|
+
config.redirects.fold(Nil: Seq[Redirect]) { sourcesToTargets =>
|
142
|
+
sourcesToTargets.foldLeft(Seq(): Seq[Redirect]) {
|
143
|
+
(redirects, sourceToTarget) =>
|
144
|
+
redirects :+ Redirect(sourceToTarget._1, sourceToTarget._2)
|
145
|
+
}
|
202
146
|
}
|
203
147
|
}
|
204
148
|
|
205
|
-
/**
|
206
|
-
* Represents a bunch of data that should be stored into an S3 objects body.
|
207
|
-
*/
|
208
|
-
case class UploadBody(
|
209
|
-
md5: MD5,
|
210
|
-
contentLength: Long,
|
211
|
-
contentEncoding: Option[String],
|
212
|
-
maxAge: Option[Int],
|
213
|
-
contentType: String,
|
214
|
-
openInputStream: () => InputStream // It's in the caller's responsibility to close this stream
|
215
|
-
)
|
216
|
-
|
217
149
|
case class S3File(s3Key: String, md5: MD5)
|
218
150
|
|
219
151
|
object S3File {
|
@@ -19,7 +19,7 @@ object Ssg {
|
|
19
19
|
}
|
20
20
|
|
21
21
|
def findSiteDirectory(workingDirectory: File): ErrorOrFile =
|
22
|
-
|
22
|
+
Files.recursiveListFiles(workingDirectory).find { file =>
|
23
23
|
file.isDirectory && automaticallySupportedSiteGenerators.exists(_.outputDirectory == file.getName)
|
24
24
|
}.fold(Left(notFoundErrorReport): ErrorOrFile)(Right(_))
|
25
25
|
}
|
@@ -2,7 +2,6 @@ package s3
|
|
2
2
|
|
3
3
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
4
4
|
import scala.concurrent.duration.{TimeUnit, Duration}
|
5
|
-
import s3.website.Utils._
|
6
5
|
import s3.website.S3.{PushSuccessReport, PushFailureReport}
|
7
6
|
import com.amazonaws.AmazonServiceException
|
8
7
|
import s3.website.model.{Config, Site}
|
@@ -18,6 +17,23 @@ package object website {
|
|
18
17
|
|
19
18
|
trait ErrorReport extends Report
|
20
19
|
|
20
|
+
object ErrorReport {
|
21
|
+
def apply(t: Throwable)(implicit logger: Logger) = new ErrorReport {
|
22
|
+
override def reportMessage = {
|
23
|
+
val extendedReport =
|
24
|
+
if (logger.verboseOutput)
|
25
|
+
Some(t.getStackTrace take 5)
|
26
|
+
else
|
27
|
+
None
|
28
|
+
s"${t.getMessage}${extendedReport.fold("")(stackTraceElems => "\n" + stackTraceElems.mkString("\n"))}"
|
29
|
+
}
|
30
|
+
}
|
31
|
+
|
32
|
+
def apply(msg: String) = new ErrorReport {
|
33
|
+
override def reportMessage = msg
|
34
|
+
}
|
35
|
+
}
|
36
|
+
|
21
37
|
trait RetrySetting {
|
22
38
|
def retryTimeUnit: TimeUnit
|
23
39
|
}
|
@@ -26,6 +42,8 @@ package object website {
|
|
26
42
|
def dryRun: Boolean
|
27
43
|
}
|
28
44
|
|
45
|
+
type S3Key = String
|
46
|
+
|
29
47
|
trait PushAction {
|
30
48
|
def actionName = getClass.getSimpleName.replace("$", "") // case object class names contain the '$' char
|
31
49
|
|
@@ -61,6 +79,8 @@ package object website {
|
|
61
79
|
|
62
80
|
type Attempt = Int
|
63
81
|
|
82
|
+
type MD5 = String
|
83
|
+
|
64
84
|
def retry[L <: Report, R](attempt: Attempt)
|
65
85
|
(createFailureReport: (Throwable) => L, retryAction: (Attempt) => Future[Either[L, R]])
|
66
86
|
(implicit retrySetting: RetrySetting, ec: ExecutionContextExecutor, logger: Logger):
|
@@ -98,4 +118,6 @@ package object website {
|
|
98
118
|
implicit def site2Config(implicit site: Site): Config = site.config
|
99
119
|
|
100
120
|
type ErrorOrFile = Either[ErrorReport, File]
|
121
|
+
|
122
|
+
lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
|
101
123
|
}
|
@@ -14,7 +14,7 @@ import scala.concurrent.duration._
|
|
14
14
|
import s3.website.S3.S3Setting
|
15
15
|
import scala.collection.JavaConversions._
|
16
16
|
import com.amazonaws.AmazonServiceException
|
17
|
-
import org.apache.commons.codec.digest.DigestUtils.
|
17
|
+
import org.apache.commons.codec.digest.DigestUtils._
|
18
18
|
import s3.website.CloudFront.CloudFrontSetting
|
19
19
|
import com.amazonaws.services.cloudfront.AmazonCloudFront
|
20
20
|
import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, CreateInvalidationRequest, TooManyInvalidationsInProgressException}
|
@@ -24,6 +24,9 @@ import java.util.concurrent.atomic.AtomicInteger
|
|
24
24
|
import org.apache.commons.io.FileUtils.{forceDelete, forceMkdir, write}
|
25
25
|
import scala.collection.mutable
|
26
26
|
import s3.website.Push.{push, CliArgs}
|
27
|
+
import s3.website.CloudFront.CloudFrontSetting
|
28
|
+
import s3.website.S3.S3Setting
|
29
|
+
import org.apache.commons.codec.digest.DigestUtils
|
27
30
|
|
28
31
|
class S3WebsiteSpec extends Specification {
|
29
32
|
|
@@ -517,6 +520,48 @@ class S3WebsiteSpec extends Specification {
|
|
517
520
|
}
|
518
521
|
}
|
519
522
|
|
523
|
+
// Because of the local database, the first and second run are implemented differently.
|
524
|
+
"pushing files for the second time" should {
|
525
|
+
"delete the S3 objects that no longer exist on the local site" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
|
526
|
+
push
|
527
|
+
setS3Files(S3File("obsolete.txt", ""))
|
528
|
+
push
|
529
|
+
sentDelete must equalTo("obsolete.txt")
|
530
|
+
}
|
531
|
+
|
532
|
+
"push new files to the bucket" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
|
533
|
+
push
|
534
|
+
setLocalFile("newfile.txt")
|
535
|
+
push
|
536
|
+
sentPutObjectRequest.getKey must equalTo("newfile.txt")
|
537
|
+
}
|
538
|
+
|
539
|
+
"push locally changed files" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
|
540
|
+
setLocalFileWithContent(("file.txt", "first run"))
|
541
|
+
push
|
542
|
+
setLocalFileWithContent(("file.txt", "second run"))
|
543
|
+
push
|
544
|
+
sentPutObjectRequests.length must equalTo(2)
|
545
|
+
}
|
546
|
+
|
547
|
+
"push locally changed files only once" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
|
548
|
+
setLocalFileWithContent(("file.txt", "first run"))
|
549
|
+
push
|
550
|
+
setS3Files(S3File("file.txt", md5Hex("first run")))
|
551
|
+
setLocalFileWithContent(("file.txt", "second run"))
|
552
|
+
push
|
553
|
+
sentPutObjectRequests.length must equalTo(2)
|
554
|
+
}
|
555
|
+
|
556
|
+
"detect files that someone else has changed on the S3 bucket" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
|
557
|
+
setLocalFileWithContent(("file.txt", "first run"))
|
558
|
+
push
|
559
|
+
setOutdatedS3Keys("file.txt")
|
560
|
+
push
|
561
|
+
sentPutObjectRequests.length must equalTo(2)
|
562
|
+
}
|
563
|
+
}
|
564
|
+
|
520
565
|
"Jekyll site" should {
|
521
566
|
"be detected automatically" in new JekyllSite with EmptySite with MockAWS with DefaultRunMode {
|
522
567
|
setLocalFile("index.html")
|
@@ -637,12 +682,6 @@ class S3WebsiteSpec extends Specification {
|
|
637
682
|
.listObjects(Matchers.any(classOf[ListObjectsRequest]))
|
638
683
|
}
|
639
684
|
|
640
|
-
def asSeenByS3Client(upload: Upload)(implicit config: Config, logger: Logger): PutObjectRequest = {
|
641
|
-
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
642
|
-
verify(amazonS3Client).putObject(req.capture())
|
643
|
-
req.getValue
|
644
|
-
}
|
645
|
-
|
646
685
|
def sentPutObjectRequests: Seq[PutObjectRequest] = {
|
647
686
|
val req = ArgumentCaptor.forClass(classOf[PutObjectRequest])
|
648
687
|
verify(amazonS3Client, Mockito.atLeast(1)).putObject(req.capture())
|
@@ -688,15 +727,14 @@ class S3WebsiteSpec extends Specification {
|
|
688
727
|
implicit final val workingDirectory: File = new File(FileUtils.getTempDirectory, "s3_website_dir" + Random.nextLong())
|
689
728
|
val siteDirectory: File // Represents the --site=X option
|
690
729
|
val configDirectory: File = workingDirectory // Represents the --config-dir=X option
|
691
|
-
|
692
|
-
lazy val allDirectories = workingDirectory :: siteDirectory :: configDirectory :: Nil
|
730
|
+
lazy val localDatabase: File = new File(FileUtils.getTempDirectory, "s3_website_local_db_" + sha256Hex(siteDirectory.getPath))
|
693
731
|
|
694
732
|
def before {
|
695
|
-
|
733
|
+
workingDirectory :: siteDirectory :: configDirectory :: Nil foreach forceMkdir
|
696
734
|
}
|
697
735
|
|
698
736
|
def after {
|
699
|
-
|
737
|
+
(workingDirectory :: siteDirectory :: configDirectory :: localDatabase :: Nil) foreach { dir =>
|
700
738
|
if (dir.exists) forceDelete(dir)
|
701
739
|
}
|
702
740
|
}
|
@@ -720,7 +758,7 @@ class S3WebsiteSpec extends Specification {
|
|
720
758
|
trait EmptySite extends Directories {
|
721
759
|
type LocalFileWithContent = (String, String)
|
722
760
|
|
723
|
-
val localFilesWithContent: mutable.
|
761
|
+
val localFilesWithContent: mutable.Set[LocalFileWithContent] = mutable.Set()
|
724
762
|
def setLocalFile(fileName: String) = setLocalFileWithContent((fileName, ""))
|
725
763
|
def setLocalFiles(fileNames: String*) = fileNames foreach setLocalFile
|
726
764
|
def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) = localFilesWithContent += fileNameAndContent
|
@@ -733,10 +771,22 @@ class S3WebsiteSpec extends Specification {
|
|
733
771
|
|s3_bucket: bucket
|
734
772
|
""".stripMargin
|
735
773
|
|
736
|
-
implicit
|
774
|
+
implicit def cliArgs: CliArgs = siteWithFilesAndContent(config, localFilesWithContent)
|
737
775
|
def pushMode: PushMode // Represents the --dry-run switch
|
738
776
|
|
739
|
-
def
|
777
|
+
private def siteWithFilesAndContent(config: String = "", localFilesWithContent: mutable.Set[LocalFileWithContent]): CliArgs = {
|
778
|
+
localFilesWithContent.foreach {
|
779
|
+
case (filePath, content) =>
|
780
|
+
val file = new File(siteDirectory, filePath)
|
781
|
+
forceMkdir(file.getParentFile)
|
782
|
+
file.createNewFile()
|
783
|
+
write(file, content)
|
784
|
+
localFilesWithContent remove(filePath, content) // Remove the file from the set once we've persisted it on the disk.
|
785
|
+
}
|
786
|
+
buildCliArgs(config)
|
787
|
+
}
|
788
|
+
|
789
|
+
private def buildCliArgs(
|
740
790
|
config: String = "",
|
741
791
|
baseConfig: String =
|
742
792
|
"""
|
@@ -762,16 +812,5 @@ class S3WebsiteSpec extends Specification {
|
|
762
812
|
override def configDir = configDirectory.getAbsolutePath
|
763
813
|
}
|
764
814
|
}
|
765
|
-
|
766
|
-
def siteWithFilesAndContent(config: String = "", localFilesWithContent: Seq[LocalFileWithContent]): CliArgs = {
|
767
|
-
localFilesWithContent.foreach {
|
768
|
-
case (filePath, content) =>
|
769
|
-
val file = new File(siteDirectory, filePath)
|
770
|
-
forceMkdir(file.getParentFile)
|
771
|
-
file.createNewFile()
|
772
|
-
write(file, content)
|
773
|
-
}
|
774
|
-
buildSite(config)
|
775
|
-
}
|
776
815
|
}
|
777
816
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_website_monadic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.32
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lauri Lehmijoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -113,14 +113,13 @@ files:
|
|
113
113
|
- src/main/java/s3/website/ByteHelper.java
|
114
114
|
- src/main/scala/s3/website/CloudFront.scala
|
115
115
|
- src/main/scala/s3/website/Diff.scala
|
116
|
+
- src/main/scala/s3/website/Logger.scala
|
116
117
|
- src/main/scala/s3/website/Push.scala
|
117
118
|
- src/main/scala/s3/website/Ruby.scala
|
118
119
|
- src/main/scala/s3/website/S3.scala
|
119
|
-
- src/main/scala/s3/website/Utils.scala
|
120
120
|
- src/main/scala/s3/website/model/Config.scala
|
121
121
|
- src/main/scala/s3/website/model/S3Endpoint.scala
|
122
122
|
- src/main/scala/s3/website/model/Site.scala
|
123
|
-
- src/main/scala/s3/website/model/errors.scala
|
124
123
|
- src/main/scala/s3/website/model/push.scala
|
125
124
|
- src/main/scala/s3/website/model/ssg.scala
|
126
125
|
- src/main/scala/s3/website/package.scala
|
@@ -1,108 +0,0 @@
|
|
1
|
-
package s3.website
|
2
|
-
|
3
|
-
import s3.website.model.Config
|
4
|
-
import scala.collection.parallel.{ForkJoinTaskSupport, ParSeq}
|
5
|
-
import scala.concurrent.forkjoin.ForkJoinPool
|
6
|
-
|
7
|
-
class Utils(implicit config: Config) {
|
8
|
-
def toParSeq[T](seq: Seq[T]): ParSeq[T] = {
|
9
|
-
val parallelSeq: ParSeq[T] = seq.par
|
10
|
-
parallelSeq.tasksupport_=(new ForkJoinTaskSupport(new ForkJoinPool(config.concurrency_level)))
|
11
|
-
parallelSeq
|
12
|
-
}
|
13
|
-
}
|
14
|
-
|
15
|
-
object Utils {
|
16
|
-
lazy val fibs: Stream[Int] = 0 #:: 1 #:: fibs.zip(fibs.tail).map { n => n._1 + n._2 }
|
17
|
-
}
|
18
|
-
|
19
|
-
class Logger(val verboseOutput: Boolean, logMessage: (String) => Unit = println) {
|
20
|
-
import Rainbow._
|
21
|
-
def debug(msg: String) = if (verboseOutput) log(Debug, msg)
|
22
|
-
def info(msg: String) = log(Info, msg)
|
23
|
-
def fail(msg: String) = log(Failure, msg)
|
24
|
-
|
25
|
-
def info(report: SuccessReport) = log(Success, report.reportMessage)
|
26
|
-
def info(report: FailureReport) = fail(report.reportMessage)
|
27
|
-
|
28
|
-
def pending(msg: String) = log(Wait, msg)
|
29
|
-
|
30
|
-
private def log(logType: LogType, msgRaw: String) {
|
31
|
-
val msg = msgRaw.replaceAll("\\n", "\n ") // Indent new lines, so that they arrange nicely with other log lines
|
32
|
-
logMessage(s"[$logType] $msg")
|
33
|
-
}
|
34
|
-
|
35
|
-
sealed trait LogType {
|
36
|
-
val prefix: String
|
37
|
-
override def toString = prefix
|
38
|
-
}
|
39
|
-
case object Debug extends LogType {
|
40
|
-
val prefix = "debg".cyan
|
41
|
-
}
|
42
|
-
case object Info extends LogType {
|
43
|
-
val prefix = "info".blue
|
44
|
-
}
|
45
|
-
case object Success extends LogType {
|
46
|
-
val prefix = "succ".green
|
47
|
-
}
|
48
|
-
case object Failure extends LogType {
|
49
|
-
val prefix = "fail".red
|
50
|
-
}
|
51
|
-
case object Wait extends LogType {
|
52
|
-
val prefix = "wait".yellow
|
53
|
-
}
|
54
|
-
}
|
55
|
-
|
56
|
-
/**
|
57
|
-
* Idea copied from https://github.com/ktoso/scala-rainbow.
|
58
|
-
*/
|
59
|
-
object Rainbow {
|
60
|
-
implicit class RainbowString(val s: String) extends AnyVal {
|
61
|
-
import Console._
|
62
|
-
|
63
|
-
/** Colorize the given string foreground to ANSI black */
|
64
|
-
def black = BLACK + s + RESET
|
65
|
-
/** Colorize the given string foreground to ANSI red */
|
66
|
-
def red = RED + s + RESET
|
67
|
-
/** Colorize the given string foreground to ANSI red */
|
68
|
-
def green = GREEN + s + RESET
|
69
|
-
/** Colorize the given string foreground to ANSI red */
|
70
|
-
def yellow = YELLOW + s + RESET
|
71
|
-
/** Colorize the given string foreground to ANSI red */
|
72
|
-
def blue = BLUE + s + RESET
|
73
|
-
/** Colorize the given string foreground to ANSI red */
|
74
|
-
def magenta = MAGENTA + s + RESET
|
75
|
-
/** Colorize the given string foreground to ANSI red */
|
76
|
-
def cyan = CYAN + s + RESET
|
77
|
-
/** Colorize the given string foreground to ANSI red */
|
78
|
-
def white = WHITE + s + RESET
|
79
|
-
|
80
|
-
/** Colorize the given string background to ANSI red */
|
81
|
-
def onBlack = BLACK_B + s + RESET
|
82
|
-
/** Colorize the given string background to ANSI red */
|
83
|
-
def onRed = RED_B+ s + RESET
|
84
|
-
/** Colorize the given string background to ANSI red */
|
85
|
-
def onGreen = GREEN_B+ s + RESET
|
86
|
-
/** Colorize the given string background to ANSI red */
|
87
|
-
def onYellow = YELLOW_B + s + RESET
|
88
|
-
/** Colorize the given string background to ANSI red */
|
89
|
-
def onBlue = BLUE_B+ s + RESET
|
90
|
-
/** Colorize the given string background to ANSI red */
|
91
|
-
def onMagenta = MAGENTA_B + s + RESET
|
92
|
-
/** Colorize the given string background to ANSI red */
|
93
|
-
def onCyan = CYAN_B+ s + RESET
|
94
|
-
/** Colorize the given string background to ANSI red */
|
95
|
-
def onWhite = WHITE_B+ s + RESET
|
96
|
-
|
97
|
-
/** Make the given string bold */
|
98
|
-
def bold = BOLD + s + RESET
|
99
|
-
/** Underline the given string */
|
100
|
-
def underlined = UNDERLINED + s + RESET
|
101
|
-
/** Make the given string blink (some terminals may turn this off) */
|
102
|
-
def blink = BLINK + s + RESET
|
103
|
-
/** Reverse the ANSI colors of the given string */
|
104
|
-
def reversed = REVERSED + s + RESET
|
105
|
-
/** Make the given string invisible using ANSI color codes */
|
106
|
-
def invisible = INVISIBLE + s + RESET
|
107
|
-
}
|
108
|
-
}
|