s3_website_monadic 0.0.35 → 0.0.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a588d29f02ef097314aa16ec55674fa6eccbe09
-  data.tar.gz: caf04222e53fdd2cacdadcc6897f9d59cfd7b850
+  metadata.gz: e7c4690fe767a7e3acb2814561d70516adb70a37
+  data.tar.gz: 58348dcb010d4f4ba30132c26e7cc8008e97f4f9
 SHA512:
-  metadata.gz: 61d6891706690b50a98966cd6860e0680ccd9e478f5a1ec2fc0c0c92a0af6e42777c546486d826177b85c73a3911eeabce9ec4b1a32f2b8a645778f99d7ad771
-  data.tar.gz: 0d49bbcd125387e1d9fcbd8346e784ff4f4957e30ecc0839fcb2714680a68c3a4bc9f7ddfacfc63238fd542d7dc013d63af61ed4a2d09d8d2cdb25cf17c273d3
+  metadata.gz: ede3a36d8687fe7d9794405810b8425c01a7ce4c5b19c196d77ee7f1737fb2913d0d970767e3d04b0c5cfa4580ca14cd5f430d55b0a532d93523d44edca504e1
+  data.tar.gz: bf26d74d0b3179386e48fd12d42ae9d58a51f7bf94b4a7ad84014513736b461c5cf4b7f2345cecbe799c611d20ee68dbd7a03a2706d9d124fdb7949187b9a681

data/.travis.yml

@@ -1,3 +1,3 @@
 language: scala
 scala:
-   2.11.0
+   2.11.1

data/README.md

@@ -17,10 +17,6 @@
 
     gem install s3_website_monadic
 
-`s3_website_monadic` requires Ruby and Java. Here is documentation on installing Ruby:
-<http://www.ruby-lang.org/en/downloads/>, and here is documentation for Java:
-<http://www.oracle.com/technetwork/java/javase/downloads/index.html>.
-
 ## Usage
 
 Here's how you can get started:
@@ -34,11 +30,6 @@ Here's how you can get started:
   S3 website. If the bucket does not exist, the command will create it for you.
 * Run `s3_website_monadic push` to push your website to S3. Congratulations! You are live.
 
-**Important security note:** if the source code of your website is publicly
-available, ensure that the `s3_website.yml` file is in the list of ignored files.
-For git users this means that the file `.gitignore` should mention the
-`s3_website.yml` file.
-
 ### For Jekyll users
 
 S3_website will automatically discover your website in the *_site* directory.
@@ -334,7 +325,7 @@ If you experience the "too many open files" error, either increase the amount of
 maximum open files (on Unix-like systems, see `man ulimit`) or decrease the
 `concurrency_level` setting.
 
-## Simulating deployments
+### Simulating deployments
 
 You can simulate the `s3_website_monadic push` operation by adding the
 `--dry-run` switch. The dry run mode will not apply any modifications on your S3
@@ -344,18 +335,31 @@ operation would actually do if run without the dry switch.
 You can use the dry run mode if you are unsure what kind of effects the `push`
 operation would cause to your live website.
 
+## Migrating from v1 to v2
+
+Please read the [release note](/changelog.md#200) on version 2. It contains
+information on backward incompatible changes.
+
 ## Example configurations
 
 See
 <https://github.com/laurilehmijoki/s3_website/blob/master/additional-docs/example-configurations.md>.
 
+## On security
+
+If the source code of your website is publicly
+available, ensure that the `s3_website.yml` file is in the list of ignored files.
+For git users this means that the file `.gitignore` should mention the
+`s3_website.yml` file.
+
+If you use the .dotenv gem, ensure that you do not push the `.env` file to a
+public git repository.
+
 ## Known issues
 
 Please create an issue and send a pull request if you spot any.
 
-## Development
-
-### Versioning
+## Versioning
 
 s3_website_monadic uses [Semantic Versioning](http://semver.org).
 
@@ -363,10 +367,9 @@ In the spirit of semantic versioning, here is the definition of public API for
 s3_website: Within a major version, s3_website_monadic will not break
 backwards-compatibility of anything that is mentioned in this README file.
 
-### Tests
+## Development
 
-  * Install bundler and run `bundle install`
-  * Run all tests by invoking `rake test`
+See [development](additional-docs/development.md).
 
 ### Contributing
 
@@ -382,12 +385,6 @@ If you are not sure how to test your pull request, you can ask the [gem owners
 However, by including proper tests, you increase the chances of your pull
 request being incorporated into future releases.
 
-#### Checklist for new features
-
-* Is it tested?
-* Is it documented in README?
-* Is it mentioned in `resources/configuration_file_template.yml`?
-
 ## License
 
 MIT. See the LICENSE file for more information.

data/additional-docs/development.md

@@ -1,4 +1,23 @@
-# Vagrant
+## Coding
+
+Install a Scala editor. Intellij IDEA has great Scala support.
+
+If you use IDEA, install the [Grep
+Console](http://plugins.jetbrains.com/plugin/?idea&pluginId=7125) plugin. It
+shows the ANSI colors in your IDEA console.
+
+### Test runs with IDEA
+
+1. Create a run profile: *Run* –> *Edit Configurations...*.
+2. Add *Application*
+3. Set *Main class* to `s3.website.Push`
+4. Set *Program arguments* to `--site=/Users/you/yourtestsite/_site --config-dir=/Users/you/yourtestsite --verbose`
+
+## Automated tests
+
+    ./sbt test
+
+## Test Linux distributions
 
 Use Vagrant for testing the installation procedure on Linux.
 
@@ -7,3 +26,4 @@ Here's howto:
 1. Install <https://www.vagrantup.com/downloads.html>
 2. `cd vagrant && vagrant status`
 3. launch with `vagrant up <name>` and ssh into with `vagrant ssh <name>`
+4. test the latest release with `gem install s3_website && s3_website push`

data/bin/s3_website_monadic

@@ -83,6 +83,11 @@ class Cli < Thor
 end
 
 def run_s3_website_jar(jar_file, call_dir, logger)
+  java_installed = resolve_exit_status('which javas') or resolve_exit_status('javas -version')
+  unless java_installed
+    logger.info_msg "Cannot find Java. s3_website push is implemented in Scala, and it needs Java to run."
+    autoinstall_java_or_print_help_and_exit(logger)
+  end
   args = ARGV.join(' ').sub('push', '')
   logger.debug_msg "Using #{jar_file}"
   if system("java -cp #{jar_file} s3.website.Push #{args}")
@@ -92,6 +97,53 @@ def run_s3_website_jar(jar_file, call_dir, logger)
   end
 end
 
+def resolve_exit_status(cmd)
+  `#{cmd}`
+  cmd_succeeded = $? == 0
+rescue
+  cmd_succeeded = false
+end
+
+def autoinstall_java_or_print_help_and_exit(logger)
+  @logger = logger
+  automatic_methods = [
+    {
+      :package_manager_lookup => 'which apt-get',
+      :install_command => 'sudo apt-get install --assume-yes openjdk-7-jre'
+    },
+    {
+      :package_manager_lookup => 'which yum',
+      :install_command => 'sudo yum install --assumeyes java-1.7.0-openjdk'
+    }
+  ]
+
+  def print_manual_method_and_exit
+    @logger.info_msg 'Go to http://java.com, install Java and then try again.'
+    exit 1
+  end
+
+  manual_method_help =
+
+  automatic_method = automatic_methods.find { |automatic_method|
+    resolve_exit_status automatic_method.fetch(:package_manager_lookup)
+  }
+  if automatic_method
+    @logger.info_msg "Do you want me to install Java with the command `#{automatic_method.fetch(:install_command)}`? [Y/n]"
+    user_answer = $stdin.gets
+    if user_answer.chomp.downcase == 'y' or user_answer == "\n"
+      automatic_method_succeeded = system automatic_method.fetch(:install_command)
+      unless automatic_method_succeeded
+        @logger.fail_msg "Could not automatically install Java. Try setting it up manually:"
+        print_manual_method_and_exit
+      end
+    else
+      print_manual_method_and_exit
+    end
+  else
+    print_manual_method_and_exit
+  end
+end
+
 def resolve_jar(project_root, logger)
   development_jar_path =
     project_root + '/target/scala-2.11/s3_website.jar'
@@ -99,11 +151,11 @@ def resolve_jar(project_root, logger)
     project_root + "/s3_website-#{S3Website::VERSION}.jar",
     (ENV['TMPDIR'] || '/tmp') + "/s3_website-#{S3Website::VERSION}.jar"
   ]
-  found_jar = ([development_jar_path] + released_jar_lookup_paths)
-  .select { |jar_path|
-    File.exists? jar_path
-  }
-  .first
+  found_jar = ([development_jar_path] + released_jar_lookup_paths).
+    select { |jar_path|
+      File.exists? jar_path
+    }.
+    first
   jar_file =
     if found_jar
       found_jar

data/build.sbt

@@ -4,7 +4,7 @@ name := "s3_website"
 
 version := "0.0.1"
 
-scalaVersion := "2.11.0"
+scalaVersion := "2.11.1"
 
 scalacOptions += "-feature"
 

data/changelog.md

@@ -8,6 +8,11 @@ This project uses [Semantic Versioning](http://semver.org).
 
 * Faster uploads for extra large sites
 
+   Use a local database for calculating diffs. This removes the need to read all
+   the files of the website, when you call the `s3_website push` command.
+
+   Use proper multithreading with JVM threads.
+
 * Simulate deployments with `push --dry-run`
 
 * Support CloudFront invalidations when the site contains over 3000 files
@@ -22,10 +27,16 @@ This project uses [Semantic Versioning](http://semver.org).
 
 * Fault tolerance – do not crash if one of the uploads fails
 
+   Before, the push command crashed if something unexpected happened. From now
+   on, s3_website will run all the operations it can, and report errors in the
+   end.
+
 ### Java is now required
 
-* The `push` command is now written in Scala. This means that you need Java 1.6
-  or above to run the command `s3_website push`.
+* The `push` command is now written in Scala
+
+   This means that you need Java 1.6 or above to run the command `s3_website
+   push`.
 
 ### Removed features
 
@@ -38,7 +49,8 @@ This project uses [Semantic Versioning](http://semver.org).
     s3_website always deletes the files that are on the S3 bucket but not on the local file system.
     Use the settings `ignore_on_server` and `exclude_from_upload` to control the retained files.
 
-* You can no longer use this gem as a Ruby library
+* You can no longer use this gem as a Ruby library. You can migrate by calling
+  the `s3_website push --site=x --config-dir=y` system command from your Ruby code.
 
 * `gzip_zopfli: true`
 

data/lib/s3_website/version.rb

@@ -1,3 +1,3 @@
 module S3Website
-  VERSION = '0.0.35'
+  VERSION = '0.0.36'
 end

data/src/main/scala/s3/website/CloudFront.scala

@@ -9,6 +9,7 @@ import s3.website.S3.{SuccessfulDelete, PushSuccessReport, SuccessfulUpload}
 import com.amazonaws.auth.BasicAWSCredentials
 import java.net.URI
 import scala.concurrent.{ExecutionContextExecutor, Future}
+import s3.website.model.Config.awsCredentials
 
 object CloudFront {
   def invalidate(invalidationBatch: InvalidationBatch, distributionId: String, attempt: Attempt = 1)
@@ -63,8 +64,7 @@ object CloudFront {
     def reportMessage = s"Failed to invalidate the CloudFront distribution (${error.getMessage})"
   }
 
-  def awsCloudFrontClient(config: Config) =
-    new AmazonCloudFrontClient(new BasicAWSCredentials(config.s3_id, config.s3_secret))
+  def awsCloudFrontClient(config: Config) = new AmazonCloudFrontClient(awsCredentials(config))
   
   def toInvalidationBatches(pushSuccessReports: Seq[PushSuccessReport])(implicit config: Config): Seq[InvalidationBatch] = {
     val invalidationPaths: Seq[String] = {

data/src/main/scala/s3/website/S3.scala

@@ -21,6 +21,7 @@ import scala.concurrent.duration.TimeUnit
 import java.util.concurrent.TimeUnit.SECONDS
 import s3.website.S3.SuccessfulUpload.humanizeUploadSpeed
 import java.io.FileInputStream
+import s3.website.model.Config.awsCredentials
 
 object S3 {
 
@@ -105,7 +106,7 @@ object S3 {
       None // We are not interested in tracking durations of PUT requests that don't contain data. Redirect is an example of such request.
   }
 
-  def awsS3Client(config: Config) = new AmazonS3Client(new BasicAWSCredentials(config.s3_id, config.s3_secret))
+  def awsS3Client(config: Config) = new AmazonS3Client(awsCredentials(config))
 
   def resolveS3Files(nextMarker: Option[String] = None, alreadyResolved: Seq[S3File] = Nil,  attempt: Attempt = 1)
                               (implicit config: Config, s3Settings: S3Setting, ec: ExecutionContextExecutor, logger: Logger, pushMode: PushMode):

data/src/main/scala/s3/website/model/Config.scala

@@ -4,10 +4,11 @@ import scala.util.{Failure, Try}
 import scala.collection.JavaConversions._
 import s3.website.Ruby.rubyRuntime
 import s3.website.ErrorReport
+import com.amazonaws.auth.{AWSCredentialsProvider, BasicAWSCredentials, DefaultAWSCredentialsProviderChain}
 
 case class Config(
-  s3_id:                      String,
-  s3_secret:                  String,
+  s3_id:                      Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
+  s3_secret:                  Option[String], // If undefined, use IAM Roles (http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-roles.html)
   s3_bucket:                  String,
   s3_endpoint:                S3Endpoint,
   max_age:                    Option[Either[Int, Map[String, Int]]],
@@ -23,6 +24,20 @@ case class Config(
 )
 
 object Config {
+
+  def awsCredentials(config: Config): AWSCredentialsProvider = {
+    val credentialsFromConfigFile = for {
+      s3_id <- config.s3_id
+      s3_secret <- config.s3_secret
+    } yield new BasicAWSCredentials(s3_id, s3_secret)
+    credentialsFromConfigFile.fold(new DefaultAWSCredentialsProviderChain: AWSCredentialsProvider)(credentials =>
+      new AWSCredentialsProvider {
+        def getCredentials = credentials
+        def refresh() = {}
+      }
+    )
+  }
+
   def loadOptionalBooleanOrStringSeq(key: String)(implicit unsafeYaml: UnsafeYaml): Either[ErrorReport, Option[Either[Boolean, Seq[String]]]] = {
     val yamlValue = for {
       optionalValue <- loadOptionalValue(key)

data/src/main/scala/s3/website/model/Site.scala

@@ -31,8 +31,8 @@ object Site {
       case Success(yamlObject) =>
         implicit val unsafeYaml = UnsafeYaml(yamlObject)
         val config: Either[ErrorReport, Config] = for {
-          s3_id <- loadRequiredString("s3_id").right
-          s3_secret <- loadRequiredString("s3_secret").right
+          s3_id <- loadOptionalString("s3_id").right
+          s3_secret <- loadOptionalString("s3_secret").right
           s3_bucket <- loadRequiredString("s3_bucket").right
           s3_endpoint <- loadEndpoint.right
           max_age <- loadMaxAge.right

data/src/test/scala/s3/website/S3WebsiteSpec.scala

@@ -591,10 +591,9 @@ class S3WebsiteSpec extends Specification {
     }
 
     "push locally unchanged files that are missing from S3" in new AllInSameDirectory with EmptySite with MockAWS with DefaultRunMode {
-      setLocalFileWithContentAndLastModified(("file.txt", "contents", new Date(1400000000000L)))
+      setLocalFileWithContent(("file.txt", "first run"))
       push
-      // Simulate the situation where someone else has deleted file.txt, and we push for the second time
-      setLocalFileWithContentAndLastModified(("file.txt", "contents", new Date(1400000000000L)))
+      removeAllFilesFromS3()
       push
       sentPutObjectRequests.length must equalTo(2) // Even though we use the local db, we should notice that someone else has deleted file.txt
     }
@@ -702,6 +701,10 @@ class S3WebsiteSpec extends Specification {
       }
     }
 
+    def removeAllFilesFromS3() {
+      setS3Files(Nil: _*) // This corresponds to the situation where the S3 bucket is empty
+    }
+
     def uploadFailsAndThenSucceeds(implicit howManyFailures: Int, callCount: AtomicInteger = new AtomicInteger(0)) {
       doAnswer(temporaryFailure(classOf[PutObjectResult]))
         .when(amazonS3Client)
@@ -795,15 +798,12 @@ class S3WebsiteSpec extends Specification {
   
   trait EmptySite extends Directories {
     type LocalFileWithContent = (String, String)
-    type LocalFileWithContentAndLastModified = (String, String, Date)
 
-    val localFilesWithContent: mutable.Set[LocalFileWithContentAndLastModified] = mutable.Set()
+    val localFilesWithContent: mutable.Set[LocalFileWithContent] = mutable.Set()
     def setLocalFile(fileName: String) = setLocalFileWithContent((fileName, ""))
     def setLocalFiles(fileNames: String*) = fileNames foreach setLocalFile
     def setLocalFileWithContent(fileNameAndContent: LocalFileWithContent) =
-      setLocalFileWithContentAndLastModified((fileNameAndContent._1, fileNameAndContent._2, new Date))
-    def setLocalFileWithContentAndLastModified(fileNameAndContentAndLastModified: LocalFileWithContentAndLastModified) =
-      localFilesWithContent += fileNameAndContentAndLastModified
+      localFilesWithContent += fileNameAndContent
     def setLocalFilesWithContent(fileNamesAndContent: LocalFileWithContent*) = fileNamesAndContent foreach setLocalFileWithContent
     var config = ""
     val baseConfig =
@@ -816,14 +816,13 @@ class S3WebsiteSpec extends Specification {
     implicit def cliArgs: CliArgs = siteWithFilesAndContent(config, localFilesWithContent)
     def pushMode: PushMode // Represents the --dry-run switch
 
-    private def siteWithFilesAndContent(config: String = "", localFilesWithContent: mutable.Set[LocalFileWithContentAndLastModified]): CliArgs = {
+    private def siteWithFilesAndContent(config: String = "", localFilesWithContent: mutable.Set[LocalFileWithContent]): CliArgs = {
       localFilesWithContent.foreach {
-        case (filePath, content, lastModified) =>
+        case (filePath, content) =>
           val file = new File(siteDirectory, filePath)
           forceMkdir(file.getParentFile)
           file.createNewFile()
           write(file, content)
-          file.setLastModified(lastModified.getTime)
       }
       localFilesWithContent.clear() // Once we've persisted the changes on the disk, clear the queue. I.e., keep the state where it should be – on the disk.
       buildCliArgs(config)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3_website_monadic
 version: !ruby/object:Gem::Version
-  version: 0.0.35
+  version: 0.0.36
 platform: ruby
 authors:
 - Lauri Lehmijoki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-28 00:00:00.000000000 Z
+date: 2014-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor