s3_website_monadic 0.0.37 → 0.0.38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/s3_website_monadic +1 -0
- data/build.sbt +0 -4
- data/lib/s3_website/version.rb +1 -1
- data/src/main/scala/s3/website/CloudFront.scala +1 -1
- data/src/main/scala/s3/website/Diff.scala +30 -33
- data/src/main/scala/s3/website/Logger.scala +5 -3
- data/src/main/scala/s3/website/Push.scala +3 -3
- data/src/main/scala/s3/website/S3.scala +56 -55
- data/src/main/scala/s3/website/model/Site.scala +1 -1
- data/src/main/scala/s3/website/model/push.scala +35 -22
- data/src/main/scala/s3/website/package.scala +2 -0
- data/src/test/scala/s3/website/S3WebsiteSpec.scala +10 -17
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: df94a2c8b44f12db2fc393cd4de9126c68ba3f17
|
4
|
+
data.tar.gz: d9cd16f493ca0e4d861a7ed2b6b176cfdf58651e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 377416d509c1df1ff449da81f87888aaf86cfec2423ddb615ac974b2c2383eef32ae1c6aee373b4ae177656fa4aebc052e95e8965131e07387c138143ffbf2b8
|
7
|
+
data.tar.gz: a89614a07d46403aaf97e29b693677ee2f704fece8836721f822fa17b728465e8c8b59e2d168f8b108e997ccc6d84708d839e8db9cdcbdc924b27418e2b055c4
|
data/bin/s3_website_monadic
CHANGED
@@ -119,6 +119,7 @@ def autoinstall_java_or_print_help_and_exit(logger)
|
|
119
119
|
|
120
120
|
def print_manual_method_and_exit
|
121
121
|
@logger.info_msg 'Go to http://java.com, install Java and then try again.'
|
122
|
+
@logger.info_msg "(If you cannot or do not want to install Java, you can use latest 1.x version of this gem, which requires only Ruby. For more info, see https://github.com/laurilehmijoki/s3_website/tree/1.x)"
|
122
123
|
exit 1
|
123
124
|
end
|
124
125
|
|
data/build.sbt
CHANGED
@@ -26,10 +26,6 @@ libraryDependencies += "org.apache.tika" % "tika-core" % "1.4"
|
|
26
26
|
|
27
27
|
libraryDependencies += "com.lexicalscope.jewelcli" % "jewelcli" % "0.8.9"
|
28
28
|
|
29
|
-
libraryDependencies += "joda-time" % "joda-time" % "2.3"
|
30
|
-
|
31
|
-
libraryDependencies += "org.joda" % "joda-convert" % "1.2"
|
32
|
-
|
33
29
|
libraryDependencies += "org.specs2" %% "specs2" % "2.3.11" % "test"
|
34
30
|
|
35
31
|
resolvers += "Sonatype OSS Snapshots" at "https://oss.sonatype.org/content/repositories/snapshots"
|
data/lib/s3_website/version.rb
CHANGED
@@ -117,7 +117,7 @@ object CloudFront {
|
|
117
117
|
|
118
118
|
|
119
119
|
def needsInvalidation: PartialFunction[PushSuccessReport, Boolean] = {
|
120
|
-
case SuccessfulUpload
|
120
|
+
case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) == FileUpdate
|
121
121
|
case SuccessfulDelete(_) => true
|
122
122
|
case _ => false
|
123
123
|
}
|
@@ -19,7 +19,7 @@ case class Diff(
|
|
19
19
|
|
20
20
|
object Diff {
|
21
21
|
|
22
|
-
type UploadBatch = Future[Either[ErrorReport, Seq[
|
22
|
+
type UploadBatch = Future[Either[ErrorReport, Seq[LocalFile]]]
|
23
23
|
|
24
24
|
def resolveDiff(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
|
25
25
|
(implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] =
|
@@ -28,26 +28,25 @@ object Diff {
|
|
28
28
|
|
29
29
|
private def resolveDiffAgainstGetBucketResponse(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
|
30
30
|
(implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
|
31
|
-
val
|
31
|
+
val diffAgainstS3 = s3FilesFuture.map { errorOrS3Files =>
|
32
32
|
errorOrS3Files.right.flatMap { s3Files =>
|
33
33
|
Try {
|
34
34
|
val s3KeyIndex = s3Files.map(_.s3Key).toSet
|
35
35
|
val s3Md5Index = s3Files.map(_.md5).toSet
|
36
36
|
val siteFiles = Files.listSiteFiles
|
37
|
-
val
|
38
|
-
val
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
.filterNot(localFile => s3Md5Index contains localFile.md5)
|
37
|
+
val existsOnS3 = (f: File) => s3KeyIndex contains site.resolveS3Key(f)
|
38
|
+
val isChangedOnS3 = (localFile: LocalFile) => !(s3Md5Index contains localFile.md5.get)
|
39
|
+
val newFiles = siteFiles collect {
|
40
|
+
case file if !existsOnS3(file) => LocalFile(file, NewFile)
|
41
|
+
}
|
42
|
+
val changedFiles = siteFiles collect {
|
43
|
+
case file if existsOnS3(file) => LocalFile(file, FileUpdate)
|
44
|
+
} filter isChangedOnS3
|
46
45
|
val unchangedFiles = {
|
47
46
|
val newOrChangedFiles = (changedFiles ++ newFiles).map(_.originalFile).toSet
|
48
47
|
siteFiles.filterNot(f => newOrChangedFiles contains f)
|
49
48
|
}
|
50
|
-
val allFiles: Seq[Either[DbRecord,
|
49
|
+
val allFiles: Seq[Either[DbRecord, LocalFile]] = unchangedFiles.map {
|
51
50
|
f => Left(DbRecord(f))
|
52
51
|
} ++ (changedFiles ++ newFiles).map {
|
53
52
|
Right(_)
|
@@ -60,8 +59,8 @@ object Diff {
|
|
60
59
|
}
|
61
60
|
}
|
62
61
|
}
|
63
|
-
def collectResult[B](pf: PartialFunction[Either[DbRecord,
|
64
|
-
|
62
|
+
def collectResult[B](pf: PartialFunction[Either[DbRecord, LocalFile],B]) =
|
63
|
+
diffAgainstS3.map { errorOrDiffSource =>
|
65
64
|
errorOrDiffSource.right map (_ collect pf)
|
66
65
|
}
|
67
66
|
val unchanged = collectResult {
|
@@ -78,16 +77,14 @@ object Diff {
|
|
78
77
|
val localKeys = for {
|
79
78
|
errorOrUnchanged <- diff.unchanged
|
80
79
|
errorsOrChanges <- Future.sequence(diff.uploads)
|
81
|
-
} yield
|
80
|
+
} yield
|
82
81
|
errorsOrChanges.foldLeft(errorOrUnchanged: Either[ErrorReport, Seq[S3Key]]) { (memo, errorOrChanges) =>
|
83
82
|
for {
|
84
83
|
mem <- memo.right
|
85
84
|
keysToDelete <- errorOrChanges.right
|
86
|
-
} yield
|
87
|
-
mem ++ keysToDelete.map(_.s3Key)
|
88
|
-
}
|
85
|
+
} yield mem ++ keysToDelete.map(_.s3Key)
|
89
86
|
}
|
90
|
-
|
87
|
+
|
91
88
|
s3Files zip localKeys map { (s3Files: Either[ErrorReport, Seq[S3File]], errorOrLocalKeys: Either[ErrorReport, Seq[S3Key]]) =>
|
92
89
|
for {
|
93
90
|
localS3Keys <- errorOrLocalKeys.right
|
@@ -115,13 +112,13 @@ object Diff {
|
|
115
112
|
|
116
113
|
def resolveDiffAgainstLocalDb(s3FilesFuture: Future[Either[ErrorReport, Seq[S3File]]])
|
117
114
|
(implicit site: Site, logger: Logger, executor: ExecutionContextExecutor): Either[ErrorReport, Diff] = {
|
118
|
-
val localDiff: Either[ErrorReport, Seq[Either[DbRecord,
|
115
|
+
val localDiff: Either[ErrorReport, Seq[Either[DbRecord, LocalFile]]] =
|
119
116
|
(for {
|
120
117
|
dbFile <- getOrCreateDbFile
|
121
118
|
databaseIndices <- loadDbFromFile(dbFile)
|
122
119
|
} yield {
|
123
120
|
val siteFiles = Files.listSiteFiles
|
124
|
-
val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord,
|
121
|
+
val recordsOrChangedFiles = siteFiles.foldLeft(Seq(): Seq[Either[DbRecord, LocalFile]]) { (localFiles, file) =>
|
125
122
|
val truncatedKey = TruncatedDbRecord(file)
|
126
123
|
val fileIsUnchanged = databaseIndices.truncatedIndex contains truncatedKey
|
127
124
|
if (fileIsUnchanged)
|
@@ -130,7 +127,7 @@ object Diff {
|
|
130
127
|
val uploadType =
|
131
128
|
if (databaseIndices.s3KeyIndex contains truncatedKey.s3Key) FileUpdate
|
132
129
|
else NewFile
|
133
|
-
localFiles :+ Right(
|
130
|
+
localFiles :+ Right(LocalFile(file, uploadType))
|
134
131
|
}
|
135
132
|
}
|
136
133
|
logger.debug(s"Discovered ${siteFiles.length} files on the local site, of which ${recordsOrChangedFiles count (_.isRight)} are new or changed")
|
@@ -145,11 +142,11 @@ object Diff {
|
|
145
142
|
case Left(f) => f
|
146
143
|
}
|
147
144
|
|
148
|
-
val
|
145
|
+
val uploadsAccordingToLocalDiff = localDiffResult collect {
|
149
146
|
case Right(f) => f
|
150
147
|
}
|
151
148
|
|
152
|
-
val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[
|
149
|
+
val changesMissedByLocalDiff: Future[Either[ErrorReport, Seq[LocalFile]]] = s3FilesFuture.map { errorOrS3Files =>
|
153
150
|
for (s3Files <- errorOrS3Files.right) yield {
|
154
151
|
val remoteS3Keys = s3Files.map(_.s3Key).toSet
|
155
152
|
val localS3Keys = unchangedAccordingToLocalDiff.map(_.s3Key).toSet
|
@@ -157,11 +154,11 @@ object Diff {
|
|
157
154
|
def isChangedOnS3(s3File: S3File) = (localS3Keys contains s3File.s3Key) && !(localMd5 contains s3File.md5)
|
158
155
|
val changedOnS3 = s3Files collect {
|
159
156
|
case s3File if isChangedOnS3(s3File) =>
|
160
|
-
|
157
|
+
LocalFile(site resolveFile s3File, FileUpdate)
|
161
158
|
}
|
162
159
|
val missingFromS3 = localS3Keys collect {
|
163
160
|
case localS3Key if !(remoteS3Keys contains localS3Key) =>
|
164
|
-
|
161
|
+
LocalFile(site resolveFile localS3Key, NewFile)
|
165
162
|
|
166
163
|
}
|
167
164
|
changedOnS3 ++ missingFromS3
|
@@ -180,14 +177,14 @@ object Diff {
|
|
180
177
|
val unchangedFilesFinal = errorOrDiffAgainstS3 map {
|
181
178
|
_ fold (
|
182
179
|
(error: ErrorReport) => Left(error),
|
183
|
-
(syncResult: (Seq[DbRecord], Seq[
|
180
|
+
(syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._1)
|
184
181
|
)
|
185
182
|
}
|
186
183
|
|
187
184
|
val changedAccordingToS3Diff = errorOrDiffAgainstS3.map {
|
188
185
|
_ fold (
|
189
186
|
(error: ErrorReport) => Left(error),
|
190
|
-
(syncResult: (Seq[DbRecord], Seq[
|
187
|
+
(syncResult: (Seq[DbRecord], Seq[LocalFile])) => Right(syncResult._2)
|
191
188
|
)
|
192
189
|
}
|
193
190
|
val persistenceError: Future[Either[ErrorReport, _]] = for {
|
@@ -198,13 +195,13 @@ object Diff {
|
|
198
195
|
records1 <- unchanged.right
|
199
196
|
records2 <- changedAccordingToS3.right
|
200
197
|
} yield
|
201
|
-
persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++
|
198
|
+
persist(records1.map(Left(_)) ++ records2.map(Right(_)) ++ uploadsAccordingToLocalDiff.map(Right(_))) match {
|
202
199
|
case Success(_) => Unit
|
203
200
|
case Failure(err) => ErrorReport(err)
|
204
201
|
}
|
205
202
|
Diff(
|
206
203
|
unchangedFilesFinal map (_.right.map(_ map (_.s3Key))),
|
207
|
-
uploads = Future(Right(
|
204
|
+
uploads = Future(Right(uploadsAccordingToLocalDiff)) :: changedAccordingToS3Diff :: Nil,
|
208
205
|
persistenceError = persistenceError map (_.left.toOption)
|
209
206
|
)
|
210
207
|
}
|
@@ -244,13 +241,13 @@ object Diff {
|
|
244
241
|
)
|
245
242
|
}
|
246
243
|
|
247
|
-
def persist(recordsOrChangedFiles: Seq[Either[DbRecord,
|
244
|
+
def persist(recordsOrChangedFiles: Seq[Either[DbRecord, LocalFile]])(implicit site: Site, logger: Logger): Try[Seq[Either[DbRecord, LocalFile]]] =
|
248
245
|
getOrCreateDbFile flatMap { dbFile =>
|
249
246
|
Try {
|
250
247
|
val dbFileContents = recordsOrChangedFiles.map { recordOrChangedFile =>
|
251
248
|
val record: DbRecord = recordOrChangedFile fold(
|
252
249
|
record => record,
|
253
|
-
changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5)
|
250
|
+
changedFile => DbRecord(changedFile.s3Key, changedFile.originalFile.length, changedFile.originalFile.lastModified, changedFile.md5.get)
|
254
251
|
)
|
255
252
|
record.s3Key :: record.fileLength :: record.fileModified :: record.uploadFileMd5 :: Nil mkString "|"
|
256
253
|
} mkString "\n"
|
@@ -278,6 +275,6 @@ object Diff {
|
|
278
275
|
|
279
276
|
object DbRecord {
|
280
277
|
def apply(original: File)(implicit site: Site): DbRecord =
|
281
|
-
DbRecord(site resolveS3Key original, original.length, original.lastModified,
|
278
|
+
DbRecord(site resolveS3Key original, original.length, original.lastModified, LocalFile.md5(original).get)
|
282
279
|
}
|
283
280
|
}
|
@@ -1,6 +1,8 @@
|
|
1
1
|
package s3.website
|
2
2
|
|
3
|
-
|
3
|
+
import scala.util.Try
|
4
|
+
|
5
|
+
class Logger(val verboseOutput: Boolean) {
|
4
6
|
def debug(msg: String) = if (verboseOutput) log(Debug, msg)
|
5
7
|
def info(msg: String) = log(Info, msg)
|
6
8
|
def fail(msg: String) = log(Failure, msg)
|
@@ -10,9 +12,9 @@ class Logger(val verboseOutput: Boolean, logMessage: (String) => Unit = println)
|
|
10
12
|
|
11
13
|
def pending(msg: String) = log(Wait, msg)
|
12
14
|
|
13
|
-
private def log(logType: LogType, msgRaw: String) {
|
15
|
+
private def log(logType: LogType, msgRaw: String): Try[Unit] = {
|
14
16
|
val msg = msgRaw.replaceAll("\\n", "\n ") // Indent new lines, so that they arrange nicely with other log lines
|
15
|
-
|
17
|
+
Try(println(s"[$logType] $msg"))
|
16
18
|
}
|
17
19
|
|
18
20
|
sealed trait LogType {
|
@@ -191,7 +191,7 @@ object Push {
|
|
191
191
|
(failureReport: PushFailureReport) => counts.copy(failures = counts.failures + 1),
|
192
192
|
(successReport: PushSuccessReport) =>
|
193
193
|
successReport match {
|
194
|
-
case succ: SuccessfulUpload => succ.
|
194
|
+
case succ: SuccessfulUpload => succ.details.fold(_.uploadType, _.uploadType) match {
|
195
195
|
case NewFile => counts.copy(newFiles = counts.newFiles + 1).addTransferStats(succ) // TODO nasty repetition here
|
196
196
|
case FileUpdate => counts.copy(updates = counts.updates + 1).addTransferStats(succ)
|
197
197
|
case RedirectFile => counts.copy(redirects = counts.redirects + 1).addTransferStats(succ)
|
@@ -228,14 +228,14 @@ object Push {
|
|
228
228
|
redirects: Int = 0,
|
229
229
|
deletes: Int = 0,
|
230
230
|
uploadedBytes: Long = 0,
|
231
|
-
uploadDurations: Seq[
|
231
|
+
uploadDurations: Seq[UploadDuration] = Nil
|
232
232
|
) {
|
233
233
|
val thereWasSomethingToPush = updates + newFiles + redirects + deletes > 0
|
234
234
|
|
235
235
|
def addTransferStats(successfulUpload: SuccessfulUpload): PushCounts =
|
236
236
|
copy(
|
237
237
|
uploadedBytes = uploadedBytes + (successfulUpload.uploadSize getOrElse 0L),
|
238
|
-
uploadDurations = uploadDurations ++ successfulUpload.uploadDuration
|
238
|
+
uploadDurations = uploadDurations ++ successfulUpload.details.fold(_.uploadDuration, _ => None)
|
239
239
|
)
|
240
240
|
}
|
241
241
|
|
@@ -2,26 +2,17 @@ package s3.website
|
|
2
2
|
|
3
3
|
import s3.website.model._
|
4
4
|
import com.amazonaws.services.s3.{AmazonS3, AmazonS3Client}
|
5
|
-
import com.amazonaws.auth.BasicAWSCredentials
|
6
5
|
import com.amazonaws.services.s3.model._
|
7
6
|
import scala.collection.JavaConversions._
|
8
7
|
import scala.concurrent.{ExecutionContextExecutor, Future}
|
9
8
|
import com.amazonaws.services.s3.model.StorageClass.ReducedRedundancy
|
10
|
-
import s3.website.S3.SuccessfulUpload
|
11
|
-
import s3.website.S3.SuccessfulDelete
|
12
|
-
import s3.website.S3.FailedUpload
|
13
|
-
import scala.Some
|
14
|
-
import s3.website.S3.FailedDelete
|
15
|
-
import s3.website.S3.S3Setting
|
16
9
|
import s3.website.ByteHelper.humanReadableByteCount
|
17
|
-
import org.joda.time.{Seconds, Duration, Interval}
|
18
|
-
import scala.concurrent.duration.TimeUnit
|
19
|
-
import java.util.concurrent.TimeUnit
|
20
10
|
import scala.concurrent.duration.TimeUnit
|
21
11
|
import java.util.concurrent.TimeUnit.SECONDS
|
22
12
|
import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
|
23
13
|
import java.io.FileInputStream
|
24
14
|
import s3.website.model.Config.awsCredentials
|
15
|
+
import scala.util.Try
|
25
16
|
|
26
17
|
object S3 {
|
27
18
|
|
@@ -29,19 +20,26 @@ object S3 {
|
|
29
20
|
(implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
|
30
21
|
upload(Right(redirect))
|
31
22
|
|
32
|
-
def uploadFile(localFile:
|
23
|
+
def uploadFile(localFile: LocalFile, a: Attempt = 1)
|
33
24
|
(implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger) =
|
34
25
|
upload(Left(localFile))
|
35
26
|
|
36
|
-
def upload(source: Either[
|
27
|
+
def upload(source: Either[LocalFile, Redirect], a: Attempt = 1)
|
37
28
|
(implicit config: Config, s3Settings: S3Setting, pushMode: PushMode, executor: ExecutionContextExecutor, logger: Logger):
|
38
29
|
Future[Either[FailedUpload, SuccessfulUpload]] =
|
39
30
|
Future {
|
40
|
-
val putObjectRequest = toPutObjectRequest(source)
|
31
|
+
val putObjectRequest = toPutObjectRequest(source).get
|
41
32
|
val uploadDuration =
|
42
33
|
if (pushMode.dryRun) None
|
43
|
-
else recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest)
|
44
|
-
val report = SuccessfulUpload(
|
34
|
+
else Some(recordUploadDuration(putObjectRequest, s3Settings.s3Client(config) putObject putObjectRequest))
|
35
|
+
val report = SuccessfulUpload(
|
36
|
+
source.fold(_.s3Key, _.s3Key),
|
37
|
+
source.fold(
|
38
|
+
localFile => Left(SuccessfulNewOrCreatedDetails(localFile.uploadType, localFile.uploadFile.get.length(), uploadDuration)),
|
39
|
+
redirect => Right(SuccessfulRedirectDetails(redirect.uploadType, redirect.redirectTarget))
|
40
|
+
),
|
41
|
+
putObjectRequest
|
42
|
+
)
|
45
43
|
logger.info(report)
|
46
44
|
Right(report)
|
47
45
|
} recoverWith retry(a)(
|
@@ -62,25 +60,30 @@ object S3 {
|
|
62
60
|
retryAction = newAttempt => this.delete(s3Key, newAttempt)
|
63
61
|
)
|
64
62
|
|
65
|
-
def toPutObjectRequest(source: Either[
|
63
|
+
def toPutObjectRequest(source: Either[LocalFile, Redirect])(implicit config: Config): Try[PutObjectRequest] =
|
66
64
|
source.fold(
|
67
|
-
localFile =>
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
md.
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
65
|
+
localFile =>
|
66
|
+
for {
|
67
|
+
uploadFile <- localFile.uploadFile
|
68
|
+
contentType <- localFile.contentType
|
69
|
+
} yield {
|
70
|
+
val md = new ObjectMetadata()
|
71
|
+
md setContentLength uploadFile.length
|
72
|
+
md setContentType contentType
|
73
|
+
localFile.encodingOnS3.map(_ => "gzip") foreach md.setContentEncoding
|
74
|
+
localFile.maxAge foreach { seconds =>
|
75
|
+
md.setCacheControl(
|
76
|
+
if (seconds == 0)
|
77
|
+
s"no-cache; max-age=$seconds"
|
78
|
+
else
|
79
|
+
s"max-age=$seconds"
|
80
|
+
)
|
81
|
+
}
|
82
|
+
val req = new PutObjectRequest(config.s3_bucket, localFile.s3Key, new FileInputStream(uploadFile), md)
|
83
|
+
config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
|
84
|
+
req
|
79
85
|
}
|
80
|
-
|
81
|
-
config.s3_reduced_redundancy.filter(_ == true) foreach (_ => req setStorageClass ReducedRedundancy)
|
82
|
-
req
|
83
|
-
},
|
86
|
+
,
|
84
87
|
redirect => {
|
85
88
|
val req = new PutObjectRequest(config.s3_bucket, redirect.s3Key, redirect.redirectTarget)
|
86
89
|
req.setMetadata({
|
@@ -93,17 +96,14 @@ object S3 {
|
|
93
96
|
md.setCacheControl("max-age=0, no-cache")
|
94
97
|
md
|
95
98
|
})
|
96
|
-
req
|
99
|
+
Try(req)
|
97
100
|
}
|
98
101
|
)
|
99
102
|
|
100
|
-
def recordUploadDuration(putObjectRequest: PutObjectRequest, f: => Unit):
|
103
|
+
def recordUploadDuration(putObjectRequest: PutObjectRequest, f: => Unit): UploadDuration = {
|
101
104
|
val start = System.currentTimeMillis()
|
102
105
|
f
|
103
|
-
|
104
|
-
Some(new Duration(start, System.currentTimeMillis))
|
105
|
-
else
|
106
|
-
None // We are not interested in tracking durations of PUT requests that don't contain data. Redirect is an example of such request.
|
106
|
+
System.currentTimeMillis - start
|
107
107
|
}
|
108
108
|
|
109
109
|
def awsS3Client(config: Config) = new AmazonS3Client(awsCredentials(config))
|
@@ -145,14 +145,18 @@ object S3 {
|
|
145
145
|
def s3Key: String
|
146
146
|
}
|
147
147
|
|
148
|
-
case class
|
148
|
+
case class SuccessfulRedirectDetails(uploadType: UploadType, redirectTarget: String)
|
149
|
+
case class SuccessfulNewOrCreatedDetails(uploadType: UploadType, uploadSize: Long, uploadDuration: Option[Long])
|
150
|
+
|
151
|
+
case class SuccessfulUpload(s3Key: S3Key,
|
152
|
+
details: Either[SuccessfulNewOrCreatedDetails, SuccessfulRedirectDetails],
|
153
|
+
putObjectRequest: PutObjectRequest)
|
149
154
|
(implicit pushMode: PushMode, logger: Logger) extends PushSuccessReport {
|
150
155
|
def reportMessage =
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
}
|
156
|
+
details.fold(
|
157
|
+
newOrCreatedDetails => s"${newOrCreatedDetails.uploadType.pushAction} $s3Key ($reportDetails)",
|
158
|
+
redirectDetails => s"${redirectDetails.uploadType.pushAction} $s3Key to ${redirectDetails.redirectTarget}"
|
159
|
+
)
|
156
160
|
|
157
161
|
def reportDetails = {
|
158
162
|
val md = putObjectRequest.getMetadata
|
@@ -169,12 +173,9 @@ object S3 {
|
|
169
173
|
}.mkString(" | ")
|
170
174
|
}
|
171
175
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
source.fold(
|
176
|
-
(localFile: LocalFileFromDisk) => Some(localFile.uploadFile.length()),
|
177
|
-
(redirect: Redirect) => None
|
176
|
+
lazy val uploadSize = details.fold(
|
177
|
+
newOrCreatedDetails => Some(newOrCreatedDetails.uploadSize),
|
178
|
+
redirectDetails => None
|
178
179
|
)
|
179
180
|
|
180
181
|
lazy val uploadSizeForHumans: Option[String] = uploadSize filter (_ => logger.verboseOutput) map humanReadableByteCount
|
@@ -182,17 +183,17 @@ object S3 {
|
|
182
183
|
lazy val uploadSpeedForHumans: Option[String] =
|
183
184
|
(for {
|
184
185
|
dataSize <- uploadSize
|
185
|
-
duration <- uploadDuration
|
186
|
+
duration <- details.left.map(_.uploadDuration).left.toOption.flatten
|
186
187
|
} yield {
|
187
188
|
humanizeUploadSpeed(dataSize, duration)
|
188
189
|
}) flatMap identity filter (_ => logger.verboseOutput)
|
189
190
|
}
|
190
191
|
|
191
192
|
object SuccessfulUpload {
|
192
|
-
def humanizeUploadSpeed(uploadedBytes: Long, uploadDurations:
|
193
|
-
val totalDurationMillis = uploadDurations.foldLeft(
|
194
|
-
memo
|
195
|
-
}
|
193
|
+
def humanizeUploadSpeed(uploadedBytes: Long, uploadDurations: UploadDuration*): Option[String] = {
|
194
|
+
val totalDurationMillis = uploadDurations.foldLeft(0L){ (memo, duration) =>
|
195
|
+
memo + duration
|
196
|
+
}
|
196
197
|
if (totalDurationMillis > 0) {
|
197
198
|
val bytesPerMillisecond = uploadedBytes / totalDurationMillis
|
198
199
|
val bytesPerSecond = bytesPerMillisecond * 1000 * uploadDurations.length
|
@@ -48,7 +48,7 @@ object Site {
|
|
48
48
|
redirects <- loadRedirects.right
|
49
49
|
} yield {
|
50
50
|
gzip_zopfli.foreach(_ => logger.info(
|
51
|
-
"""|
|
51
|
+
"""|Zopfli is not currently supported. Falling back to regular gzip.
|
52
52
|
|If you find a stable Java implementation for zopfli, please send an email to lauri.lehmijoki@iki.fi about it."""
|
53
53
|
.stripMargin))
|
54
54
|
extensionless_mime_type.foreach(_ => logger.info(
|
@@ -7,10 +7,11 @@ import java.util.zip.GZIPOutputStream
|
|
7
7
|
import org.apache.tika.Tika
|
8
8
|
import s3.website.Ruby._
|
9
9
|
import s3.website._
|
10
|
-
import s3.website.model.
|
10
|
+
import s3.website.model.LocalFile.tika
|
11
11
|
import s3.website.model.Encoding.encodingOnS3
|
12
12
|
import java.io.File.createTempFile
|
13
13
|
import org.apache.commons.io.IOUtils.copy
|
14
|
+
import scala.util.Try
|
14
15
|
|
15
16
|
object Encoding {
|
16
17
|
|
@@ -34,27 +35,34 @@ object Encoding {
|
|
34
35
|
}
|
35
36
|
}
|
36
37
|
|
37
|
-
sealed trait UploadType
|
38
|
+
sealed trait UploadType {
|
39
|
+
val pushAction: PushAction
|
40
|
+
}
|
41
|
+
|
42
|
+
case object NewFile extends UploadType {
|
43
|
+
val pushAction = Created
|
44
|
+
}
|
45
|
+
case object FileUpdate extends UploadType {
|
46
|
+
val pushAction = Updated
|
47
|
+
}
|
38
48
|
|
39
|
-
case object
|
40
|
-
|
41
|
-
|
49
|
+
case object RedirectFile extends UploadType {
|
50
|
+
val pushAction = Redirected
|
51
|
+
}
|
42
52
|
|
43
|
-
case class
|
53
|
+
case class LocalFile(originalFile: File, uploadType: UploadType)(implicit site: Site) {
|
44
54
|
lazy val s3Key = site.resolveS3Key(originalFile)
|
45
55
|
|
46
56
|
lazy val encodingOnS3 = Encoding.encodingOnS3(s3Key)
|
47
57
|
|
48
|
-
lazy val lastModified = originalFile.lastModified
|
49
|
-
|
50
58
|
/**
|
51
59
|
* This is the file we should upload, because it contains the potentially gzipped contents of the original file.
|
52
60
|
*
|
53
61
|
* May throw an exception, so remember to call this in a Try or Future monad
|
54
62
|
*/
|
55
|
-
lazy val uploadFile: File =
|
63
|
+
lazy val uploadFile: Try[File] = LocalFile uploadFile originalFile
|
56
64
|
|
57
|
-
lazy val contentType = {
|
65
|
+
lazy val contentType: Try[String] = tika map { tika =>
|
58
66
|
val mimeType = tika.detect(originalFile)
|
59
67
|
if (mimeType.startsWith("text/") || mimeType == "application/json")
|
60
68
|
mimeType + "; charset=utf-8"
|
@@ -84,24 +92,29 @@ case class LocalFileFromDisk(originalFile: File, uploadType: UploadType)(implici
|
|
84
92
|
/**
|
85
93
|
* May throw an exception, so remember to call this in a Try or Future monad
|
86
94
|
*/
|
87
|
-
lazy val md5 =
|
95
|
+
lazy val md5 = LocalFile md5 originalFile
|
88
96
|
}
|
89
97
|
|
90
|
-
object
|
91
|
-
lazy val tika = new Tika()
|
98
|
+
object LocalFile {
|
99
|
+
lazy val tika = Try(new Tika())
|
92
100
|
|
93
|
-
def md5(originalFile: File)(implicit site: Site) =
|
101
|
+
def md5(originalFile: File)(implicit site: Site): Try[MD5] =
|
102
|
+
uploadFile(originalFile) map { file =>
|
103
|
+
using(fis { file }) { DigestUtils.md5Hex }
|
104
|
+
}
|
94
105
|
|
95
|
-
def uploadFile(originalFile: File)(implicit site: Site): File =
|
106
|
+
def uploadFile(originalFile: File)(implicit site: Site): Try[File] =
|
96
107
|
encodingOnS3(site resolveS3Key originalFile)
|
97
|
-
.fold(originalFile)(algorithm =>
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
108
|
+
.fold(Try(originalFile))(algorithm =>
|
109
|
+
Try {
|
110
|
+
val tempFile = createTempFile(originalFile.getName, "gzip")
|
111
|
+
tempFile.deleteOnExit()
|
112
|
+
using(new GZIPOutputStream(new FileOutputStream(tempFile))) { stream =>
|
113
|
+
copy(fis(originalFile), stream)
|
114
|
+
}
|
115
|
+
tempFile
|
102
116
|
}
|
103
|
-
|
104
|
-
})
|
117
|
+
)
|
105
118
|
|
106
119
|
private[this] def fis(file: File): InputStream = new FileInputStream(file)
|
107
120
|
private[this] def using[T <: Closeable, R](cl: T)(f: (T) => R): R = try f(cl) finally cl.close()
|
@@ -21,13 +21,15 @@ import com.amazonaws.services.cloudfront.model.{CreateInvalidationResult, Create
|
|
21
21
|
import org.mockito.stubbing.Answer
|
22
22
|
import org.mockito.invocation.InvocationOnMock
|
23
23
|
import java.util.concurrent.atomic.AtomicInteger
|
24
|
-
import org.apache.commons.io.FileUtils.
|
24
|
+
import org.apache.commons.io.FileUtils._
|
25
25
|
import scala.collection.mutable
|
26
26
|
import s3.website.Push.{push, CliArgs}
|
27
27
|
import s3.website.CloudFront.CloudFrontSetting
|
28
28
|
import s3.website.S3.S3Setting
|
29
29
|
import org.apache.commons.codec.digest.DigestUtils
|
30
30
|
import java.util.Date
|
31
|
+
import s3.website.CloudFront.CloudFrontSetting
|
32
|
+
import s3.website.S3.S3Setting
|
31
33
|
|
32
34
|
class S3WebsiteSpec extends Specification {
|
33
35
|
|
@@ -799,12 +801,15 @@ class S3WebsiteSpec extends Specification {
|
|
799
801
|
trait EmptySite extends Directories {
|
800
802
|
type LocalFileWithContent = (String, String)
|
801
803
|
|
802
|
-
val localFilesWithContent: mutable.Set[LocalFileWithContent] = mutable.Set()
|
803
804
|
def setLocalFile(fileName: String) = setLocalFileWithContent((fileName, ""))
|
804
805
|
def setLocalFiles(fileNames: String*) = fileNames foreach setLocalFile
|
805
|
-
def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) =
|
806
|
-
localFilesWithContent += fileNameAndContent
|
807
806
|
def setLocalFilesWithContent(fileNamesAndContent: LocalFileWithContent*) = fileNamesAndContent foreach setLocalFileWithContent
|
807
|
+
def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) = {
|
808
|
+
val file = new File(siteDirectory, fileNameAndContent._1)
|
809
|
+
forceMkdir(file.getParentFile)
|
810
|
+
file.createNewFile()
|
811
|
+
write(file, fileNameAndContent._2)
|
812
|
+
}
|
808
813
|
var config = ""
|
809
814
|
val baseConfig =
|
810
815
|
"""
|
@@ -813,21 +818,9 @@ class S3WebsiteSpec extends Specification {
|
|
813
818
|
|s3_bucket: bucket
|
814
819
|
""".stripMargin
|
815
820
|
|
816
|
-
implicit def cliArgs: CliArgs =
|
821
|
+
implicit def cliArgs: CliArgs = buildCliArgs(config)
|
817
822
|
def pushMode: PushMode // Represents the --dry-run switch
|
818
823
|
|
819
|
-
private def siteWithFilesAndContent(config: String = "", localFilesWithContent: mutable.Set[LocalFileWithContent]): CliArgs = {
|
820
|
-
localFilesWithContent.foreach {
|
821
|
-
case (filePath, content) =>
|
822
|
-
val file = new File(siteDirectory, filePath)
|
823
|
-
forceMkdir(file.getParentFile)
|
824
|
-
file.createNewFile()
|
825
|
-
write(file, content)
|
826
|
-
}
|
827
|
-
localFilesWithContent.clear() // Once we've persisted the changes on the disk, clear the queue. I.e., keep the state where it should be – on the disk.
|
828
|
-
buildCliArgs(config)
|
829
|
-
}
|
830
|
-
|
831
824
|
private def buildCliArgs(
|
832
825
|
config: String = "",
|
833
826
|
baseConfig: String =
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_website_monadic
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.38
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Lauri Lehmijoki
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-06-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|