middleman-s3_sync 3.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in middleman-s3_sync.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Frederic Jean
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# Middleman::S3Sync
+
+This gem determines which files need to be added, updated and optionally deleted
+and only transfer these files up. This reduces the impact of an update
+on a web site hosted on S3.
+
+## Why not Middleman Sync?
+
+[Middleman Sync](https://github.com/karlfreeman/middleman-sync) does a
+great job to push [Middleman](http://middlemanapp.com)  generated
+websites to S3. The only issue I have with it is that it pushes
+every files under build to S3 and doesn't seem to properly delete files
+that are no longer needed.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'middleman-s3_sync'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install middleman-s3_sync
+
+## Usage
+
+You need to add the following code to your ```config.rb``` file:
+
+```ruby
+activate :s3_sync do |s3_sync|
+  s3_sync.bucket                = 'my.bucket.com' # The name of the S3 bucket you are targetting. This is globally unique.
+  s3_sync.region                = 'us-west-1'     # The AWS region for your bucket.
+  s3_sync.aws_access_key_id     = 'AWS KEY ID'
+  s3_sync.aws_secret_access_key = 'AWS SECRET KEY'
+  s3_sync.delete                = false # We delete stray files by default.
+  s3_sync.after_build           = false # We chain after the build step by default. This may not be your desired behavior...
+end
+```
+
+You can then start synchronizing files with S3 through ```middleman s3_sync```. 
+
+## A Debt of Gratitude
+
+I used Middleman Sync as a template for building a Middleman extension.
+The code is well structured and easy to understand and it was easy to
+extend it to add my synchronization code. My gratitude goes to @karlfreeman
+and is work on Middleman sync.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/middleman-s3_sync.rb

@@ -0,0 +1,106 @@
+require 'middleman-core'
+require 'fog'
+require 'parallel'
+require 'digest/md5'
+require 'middleman-s3_sync/version'
+require 'middleman-s3_sync/commands'
+
+::Middleman::Extensions.register(:s3_sync, '>= 3.0.0') do
+  require 'middleman-s3_sync/extension'
+  ::Middleman::S3Sync
+end
+
+module Middleman
+  module S3Sync
+    class << self
+      def sync
+        puts "Gathering local files."
+        local_files = Dir[options.public_path + "/**/*"]
+          .reject { |f| File.directory?(f) }
+          .map { |f| f.gsub(/^build\//, '') }
+        puts "Gathering remote files."
+        remote_files = bucket.files.map { |f| f.key }
+
+        # First pass on the set of files to work with.
+        puts "Determine files to add to S3."
+        files_to_push = local_files - remote_files
+        if options.delete
+          puts "Determine which files to delete from S3"
+          files_to_delete = remote_files - local_files
+        end
+        files_to_evaluate = local_files - files_to_push
+
+        # No need to evaluate the files that are newer on S3 than the local files.
+        puts "Determine which local files are newer than their S3 counterparts"
+        files_to_reject = []
+        Parallel.each(files_to_evaluate, :in_threads => 4) do |f|
+          print '.'
+          local_mtime = File.mtime("build/#{f}")
+          remote_mtime = s3_files.get(f).last_modified
+          files_to_reject << f if remote_mtime >= local_mtime
+        end
+
+        files_to_evaluate = files_to_evaluate - files_to_reject
+
+        # Are the files different? Use MD5 to see
+        if (files_to_evaluate.size > 0)
+          puts "\n\nDetermine which remaining files are actually different than their S3 counterpart."
+          Parallel.each(files_to_evaluate, :in_threads => 4) do |f|
+            print '.'
+            local_md5 = Digest::MD5.hexdigest(File.read("build/#{f}"))
+            remote_md5 = s3_files.get(f).etag
+            files_to_push << f if local_md5 != remote_md5
+          end
+        end
+
+        if files_to_push.size > 0
+          puts "\n\nReady to apply updates to S3."
+          files_to_push.each do |f|
+            if remote_files.include?(f)
+              puts "Updating #{f}"
+              file = s3_files.get(f)
+              file.body = File.open("build/#{f}")
+              file.save
+            else
+              puts "Creating #{f}"
+              file = bucket.files.create({
+                :key => f,
+                :body => File.open("build/#{f}"),
+                :public => true
+              })
+            end
+          end
+        else
+          puts "\n\nNo files to update."
+        end
+
+        if options.delete
+          files_to_delete.each do |f|
+            puts "Deleting #{f}"
+            file = s3_files.get(f)
+            file.destroy
+          end
+        end
+      end
+
+      protected
+      def connection
+        @connection ||= Fog::Storage.new({
+          :provider => 'AWS',
+          :aws_access_key_id => options.aws_access_key_id,
+          :aws_secret_access_key => options.aws_secret_access_key,
+          :region => options.region
+        })
+      end
+
+      def bucket
+        @bucket ||= connection.directories.get(options.bucket)
+      end
+
+      def s3_files
+        @s3_files ||= bucket.files
+      end
+    end
+  end
+end
+

data/lib/middleman-s3_sync/commands.rb

@@ -0,0 +1,31 @@
+require 'middleman-core/cli'
+require 'middleman-s3_sync/extension'
+
+module Middleman
+  module Cli
+    class S3Sync < Thor
+      include Thor::Actions
+
+      check_unknown_options!
+
+      namespace :s3_sync
+
+      def self.exit_on_failure?
+        true
+      end
+
+      desc "s3_sync", "Builds and push the minimum set of files needed to S3"
+      def s3_sync
+        shared_inst = ::Middleman::Application.server.inst
+        bucket = shared_inst.options.bucket
+        if (!bucket)
+          raise Thor::Error.new "You need to activate this extension."
+        end
+        
+        ::Middleman::S3Sync.sync
+
+      end
+
+    end
+  end
+end

data/lib/middleman-s3_sync/extension.rb

@@ -0,0 +1,48 @@
+require 'middleman-core'
+
+module Middleman
+  module S3Sync
+    class Options < Struct.new(
+      :prefix,
+      :public_path,
+      :bucket,
+      :region,
+      :aws_access_key_id,
+      :aws_secret_access_key,
+      :after_build,
+      :delete,
+      :existing_remote_file,
+    )
+    end
+
+    class << self
+      def options
+        @@options
+      end
+
+      def registered(app, options_hash = {}, &block)
+        options = Options.new(options_hash)
+        yield options if block_given?
+
+        @@options = options
+
+        app.send :include, Helpers
+
+        options.public_path ||= "build"
+        app.after_configuration do |config|
+          after_build do |builder|
+            ::Middleman::S3Sync.sync if options.after_build
+          end
+        end
+      end
+
+      alias :included :registered
+
+      module Helpers
+        def options
+          ::Middleman::S3Sync.options
+        end
+      end
+    end
+  end
+end

data/lib/middleman-s3_sync/version.rb

@@ -0,0 +1,5 @@
+module Middleman
+  module S3Sync
+    VERSION = "3.0.0"
+  end
+end

data/lib/middleman_extension.rb

@@ -0,0 +1 @@
+require 'middleman-s3_sync'

data/middleman-s3_sync.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'middleman-s3_sync/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "middleman-s3_sync"
+  gem.version       = Middleman::S3Sync::VERSION
+  gem.authors       = ["Frederic Jean"]
+  gem.email         = ["fred@fredjean.net"]
+  gem.description   = %q{Only syncs files that have been updated to S3.}
+  gem.summary       = %q{Tries really, really hard not to push files to S3.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency 'middleman-core', '>= 3.0.0'
+  gem.add_runtime_dependency 'fog'
+  gem.add_runtime_dependency 'parallel'
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: middleman-s3_sync
+version: !ruby/object:Gem::Version
+  prerelease: 
+  version: 3.0.0
+platform: ruby
+authors:
+- Frederic Jean
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: middleman-core
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: fog
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: parallel
+  prerelease: false
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Only syncs files that have been updated to S3.
+email:
+- fred@fredjean.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/middleman-s3_sync.rb
+- lib/middleman-s3_sync/commands.rb
+- lib/middleman-s3_sync/extension.rb
+- lib/middleman-s3_sync/version.rb
+- lib/middleman_extension.rb
+- middleman-s3_sync.gemspec
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: 3576862975289207856
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: 3576862975289207856
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Tries really, really hard not to push files to S3.
+test_files: []