hem 1.0.1.beta1

data/lib/hem/lib/host_check.rb

@@ -0,0 +1,34 @@
+module Hem
+  module Lib
+    module HostCheck
+      class << self
+        include Hem::Lib::HostCheck
+        def check opts = {}
+          opts = {
+            :filter => nil,
+            :raise => false
+          }.merge(opts)
+
+          results = {}
+          methods = Hem::Lib::HostCheck.public_instance_methods(false)
+          methods.each do |method|
+            next if opts[:filter] && !method.match(opts[:filter])
+
+            if opts[:raise]
+              self.send method, opts
+            else
+              begin
+                self.send method, opts
+                results[method] = :ok
+              rescue Hem::Error => error
+                results[method] = error
+              end
+            end
+          end
+
+          return results
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/s3/local/file.rb

@@ -0,0 +1,40 @@
+module Hem
+  module Lib
+    module S3
+      module Local
+        class File
+          def initialize file
+            @file = file
+          end
+
+          def buffer
+            # NOP
+          end
+
+          def size
+            @file.size
+          end
+
+          def close
+            @file.close
+          end
+
+          def read_io
+            @file
+          end
+
+          def copy_from io, opts = {}, &block
+            begin
+              while (data = io.readpartial 16984) do
+                @file.write data
+                yield data if block_given?
+              end
+            rescue EOFError
+              # NOP
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/s3/local/iohandler.rb

@@ -0,0 +1,36 @@
+module Hem
+  module Lib
+    module S3
+      module Local
+        class IoHandler
+          include Hem::Logging
+
+          def initialize path
+            @path = path
+          end
+
+          def ls
+            logger.debug("s3sync: Listing local directory: #{@path}")
+            out = {}
+            dir = "#{@path.chomp('/')}/"
+            files = Dir.glob("#{dir}**/*")
+            files.each do |file|
+              out[file.gsub(/^#{dir}/, '')] = Digest::MD5.file(file).hexdigest
+            end
+            return out
+          end
+
+          def open file, mode
+            file_path = ::File.join(@path, file)
+            FileUtils.mkdir_p ::File.dirname(file_path)
+            File.new ::File.open(file_path, mode)
+          end
+
+          def rm file
+            ::File.unlink ::File.join(@path, file)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/s3/remote/file.rb

@@ -0,0 +1,57 @@
+module Hem
+  module Lib
+    module S3
+      module Remote
+        class File
+          def initialize object, prefix
+            @object = object
+            @prefix = prefix
+            @r_buffer, @w_buffer = IO.pipe
+            @buffer_thread = nil
+          end
+
+          def buffer
+            @buffer_thread = Thread.new do
+              @object.get do |chunk|
+                @w_buffer.write chunk
+              end
+              @w_buffer.close # required for EOF
+            end
+          end
+
+          def size
+            @object.content_length
+          end
+
+          def close
+            @r_buffer.close unless @r_buffer.closed?
+            @w_buffer.close unless @w_buffer.closed?
+            @buffer_thread.exit if @buffer_thread
+          end
+
+          def read_io
+            @r_buffer
+          end
+
+          # This is a bit nasty but it gets the job done
+          def copy_from io, opts = {}, &block
+            ninety_gb = 1024 * (90000000) # arbitrarily high number
+            opts[:multipart_threshold] = ninety_gb
+
+            if block_given?
+              io.instance_eval "
+                def read bytes
+                  data = super bytes
+                  @block.call(data)
+                  data
+                end"
+              io.instance_variable_set '@block', block
+            end
+
+            @object.upload_file(io, opts)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/s3/remote/iohandler.rb

@@ -0,0 +1,38 @@
+module Hem
+  module Lib
+    module S3
+      module Remote
+        class IoHandler
+          include Hem::Logging
+
+          def initialize s3, bucket, prefix
+            @s3 = s3
+            @bucket = bucket
+            @prefix = prefix ? "#{prefix.gsub(/^\//, '').chomp('/')}/" : ""
+          end
+
+          def ls
+            out = {}
+            logger.debug("s3sync: Listing remote bucket: #{@bucket} w/ prefix #{@prefix}")
+            @s3.bucket(@bucket).objects(:prefix => @prefix).each do |file|
+              filename = file.key.gsub(/^#{@prefix}/, '')
+              next if filename == ""
+              out[filename] = file.etag.gsub('"', '')
+            end
+            return out
+          end
+
+          def open file, mode
+            s3_key = ::File.join(@prefix, file)
+            File.new @s3.bucket(@bucket).object(s3_key), @prefix
+          end
+
+          def rm file
+            s3_key = ::File.join(@prefix, file)
+            @s3.bucket(@bucket).object(s3_key).delete
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/s3/sync.rb

@@ -0,0 +1,134 @@
+module Hem
+  module Lib
+    module S3
+      class Sync
+        include Hem::Logging
+
+        def initialize opts = {}
+          require 'aws-sdk'
+
+          @opts = {
+            :access_key_id => nil,
+            :secret_access_key => nil,
+            :region => 'eu-west-1',
+            :retry_limit => 15
+          }.merge(opts)
+
+          if Hem.windows?
+            Aws.config[:ssl_ca_bundle] = File.expand_path('../../../../../ssl/ca-bundle-s3.crt', __FILE__)
+          end
+
+          handle_s3_error do
+            # AWS::S3 is flakey about actually raising this error when nil is provided
+            [:access_key_id, :secret_access_key].each do |k|
+              raise Aws::Errors::MissingCredentialsError if @opts[k].nil?
+            end
+          end
+
+          logger.debug("s3sync: Options #{@opts}")
+        end
+
+        def sync source, dest, opts = {}
+          delta = {:add => [], :remove => []}
+
+          handle_s3_error do
+            opts = {
+              :delete => true,
+              :dry => false,
+              :progress => Hem.method(:progress)
+            }.merge(opts)
+
+            source_io = io_handler(source)
+            destination_io = io_handler(dest)
+
+            logger.debug("s3sync: Synchronzing (#{source_io.class.name} -> #{destination_io.class.name}")
+
+            raise "S3 -> S3 synchronisation not supported" if source_io.is_a? Remote and destination_io.is_a? Remote
+
+            source_listing = source_io.ls
+            destination_listing = destination_io.ls
+            logger.debug("s3sync: Source listing - #{source_listing}")
+            logger.debug("s3sync: Destination listing - #{destination_listing}")
+
+            delta = delta(source_listing, destination_listing)
+            logger.debug("s3sync: Delta #{delta}")
+            break if opts[:dry]
+
+            delta[:add].each do |file|
+              logger.debug("s3sync: Synchronizing #{file}")
+              source_file = source_io.open(file, "r")
+              destination_file = destination_io.open(file, "wb+")
+
+              source_file.buffer
+
+              size = source_file.size
+              destination_file.copy_from(source_file.read_io, :content_length => size) do |chunk|
+                opts[:progress].call(file, (chunk || '').length, size, :update)
+              end
+
+              destination_file.close
+              source_file.close
+
+              opts[:progress].call(file, 0, size, :finish)
+            end
+            break unless opts[:delete]
+
+            delta[:remove].each do |file|
+              logger.debug("s3sync: Removing #{file}")
+              destination_io.rm(file)
+            end
+          end
+
+          return delta
+        end
+
+        private
+
+        def s3
+          @s3 ||= Aws::S3::Resource.new @opts
+        end
+
+        def handle_s3_error
+          exception = Hem::Error.new("Could not sync assets")
+          begin
+            yield
+          rescue Errno::ENETUNREACH
+            Hem.ui.error "  Could not contact Amazon servers."
+            Hem.ui.error "  This can sometimes be caused by missing AWS credentials"
+            raise exception
+          rescue Aws::S3::Errors::NoSuchBucket
+            Hem.ui.error "  Asset bucket #{Hem.project_config.asset_bucket} does not exist!"
+            # We allow this one to be skipped as there are obviously no assets to sync
+          rescue Aws::S3::Errors::AccessDenied
+            Hem.ui.error "  Your AWS key does not have access to the #{Hem.project_config.asset_bucket} S3 bucket!"
+            Hem.ui.error "  Please request access to this bucket from your TTL or via an internal support request"
+            raise exception
+          rescue Aws::Errors::MissingCredentialsError
+            Hem.ui.warning "  AWS credentials not set!"
+            Hem.ui.warning "  Please request credentials from internalsupport@inviqa.com or in #devops and configure them with `hem config`"
+            raise exception
+          end
+        end
+
+        def delta source, dest
+          to_add = (source.sort - dest.sort).map(&:first)
+          to_remove = (dest.sort - source.sort).map(&:first)
+          to_remove = to_remove - to_add
+
+          {
+            :add => to_add,
+            :remove => to_remove
+          }
+        end
+
+        def io_handler uri
+          require 'uri'
+          parsed = URI.parse(uri)
+          parsed.scheme == 's3' ?
+            Remote::IoHandler.new(s3, parsed.host, parsed.path) :
+            Local::IoHandler.new(uri)
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/seed/project.rb

@@ -0,0 +1,71 @@
+module Hem
+  module Lib
+    module Seed
+      class Project
+        def initialize(opts = {})
+          @opts = {
+            :replacer => Replacer.new,
+            :config_class => Hem::Config::File,
+            :project_config_file => Hem.project_config_file,
+            :ssl_cert_generator => Hem::Lib::SelfSignedCertGenerator
+          }.merge! opts
+        end
+
+        def setup seed, config
+          seed.update
+          seed.export config[:project_path], config
+          config[:seed][:version] = seed.version
+          config[:hostname] = "#{config[:name]}.dev"
+          config[:asset_bucket] = "inviqa-assets-#{config[:name]}"
+          config[:vm] = {
+            :project_mount_path => "/vagrant"
+          }
+          config[:ssl] = @opts[:ssl_cert_generator].generate config[:hostname]
+          config[:chef_ssl] = {}
+          config[:ssl].each do |k, v|
+            config[:chef_ssl][k] = v.gsub("\n", "\\n")
+          end
+
+          @opts[:replacer].replace(config[:project_path], config)
+          load_seed_init(config)
+
+          project_path = config[:project_path]
+          config.delete :project_path
+          config.delete :ssl
+          config.delete :chef_ssl
+          @opts[:config_class].save @opts[:project_config_file], config
+
+          initialize_git project_path, config[:git_url]
+        end
+
+        private
+
+        def load_seed_init config
+          Hem.project_config = DeepStruct.wrap(config)
+          seed_init_file = File.join(config[:project_path], 'seedinit.rb')
+          if File.exists?(seed_init_file)
+            require seed_init_file
+            File.unlink(seed_init_file)
+          end
+        end
+
+        def initialize_git path, git_url
+          Dir.chdir path do
+            Hem::Helper.shell 'git', 'init'
+            Hem::Helper.shell 'git', 'add', '--all'
+            Hem::Helper.shell 'git', 'commit', '-m', "'Initial hem project'"
+            Hem::Helper.shell 'git', 'checkout', '-b', 'develop'
+
+            # Github for windows gets clever adding origin / upstream remotes in system level gitconfig
+            # :facepalm:
+            begin
+              Hem::Helper.shell 'git', 'remote', 'add', 'origin', git_url
+            rescue Hem::ExternalCommandError
+              Hem::Helper.shell 'git', 'remote', 'set-url', 'origin', git_url
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/seed/replacer.rb

@@ -0,0 +1,56 @@
+module Hem
+  module Lib
+    module Seed
+      class Replacer
+        # Matching files/directories to be excluded from replacements
+        EXCLUDES = ["\\.git/", "^./bin", "^./lib", "^./spec"]
+
+        def replace(path, tokens)
+          if tokens.instance_of? Hash
+            tokens = flat_hash(tokens)
+          elsif !tokens.instance_of? Array
+            raise "Invalid token list (expected Array or Hash)"
+          end
+
+          return search_replace(path, tokens)
+        end
+
+        private
+
+        def search_replace(path, tokens, &block)
+          require 'find'
+          files = []
+          excludes = Regexp.new(EXCLUDES.join("|"))
+          Find.find(path) do |candidate|
+            Find.prune if candidate =~ excludes  # Skip excluded
+            next unless FileTest.file? candidate # Skip unless file
+
+            content = File.read(candidate)
+            next unless content.force_encoding("UTF-8").valid_encoding? # Skip unless file can be valid UTF-8
+
+            match = false
+            tokens.each do |token, replacement|
+              token = "{{#{token.join('.')}}}"
+              match = content.match(token)
+              if match
+                content.gsub!(token, replacement)
+                files.push(candidate)
+              end
+            end
+
+            File.write(candidate, content) if files.include? candidate
+          end
+          return files.uniq
+        end
+
+        # http://stackoverflow.com/questions/9647997/converting-a-nested-hash-into-a-flat-hash
+        def flat_hash(hash, k = [])
+          return {k => hash} unless hash.is_a?(Hash)
+          hash.inject({}) do |h, v|
+            h.merge! flat_hash(v[-1], k + [v[0]])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/seed/seed.rb

@@ -0,0 +1,111 @@
+module Hem
+  module Lib
+    module Seed
+      class Seed
+        include Hem::Logging
+
+        def initialize(seed_path, url)
+          @seed_path = seed_path
+          @url = url
+        end
+
+        def tags
+          tags = []
+          Dir.chdir @seed_path do
+            tag_output = Hem::Helper.shell "git tag", :capture => true
+            tags = tag_output.split("\n")
+          end
+          tags
+        end
+
+        def export path, opts = {}
+          opts = {
+            :name => 'seed',
+            :ref => 'master'
+          }.merge(opts)
+
+          path = File.expand_path(path)
+          FileUtils.mkdir_p path
+
+          Hem.ui.success "Exporting #{opts[:name]} to #{path}"
+
+          tmp_path = Dir.mktmpdir("hem-seed-export")
+
+          Dir.chdir @seed_path do
+            Hem::Helper.shell "git clone --branch #{opts[:ref].shellescape} . #{tmp_path}"
+          end
+
+          Dir.chdir tmp_path do
+            Hem::Helper.shell "git archive #{opts[:ref].shellescape} | tar -x -C #{path.shellescape}"
+
+            submodules = Hem::Helper.shell "git submodule status", :capture => true, :strip => false
+
+            next if submodules.empty?
+
+            Hem.ui.success "Cloning submodules"
+            Hem::Helper.shell "git submodule update --init", :realtime => true, :indent => 2
+
+            # Export submodules
+            # git submodule foreach does not play nice on windows so we fake it here
+            submodules.split("\n").each do |line|
+              matches = line.match /^[\s-][a-z0-9]+ (.+)/
+              next unless matches
+              submodule_path = matches[1]
+              Hem.ui.success "Exporting '#{submodule_path}' #{opts[:name]} submodule"
+              Hem::Helper.shell "cd #{tmp_path}/#{submodule_path.shellescape} && git archive HEAD | tar -x -C #{path}/#{submodule_path.shellescape}"
+            end
+          end
+
+          FileUtils.rm_f tmp_path
+        end
+
+        def update
+          Hem.ui.success "Fetching / Updating seed"
+          FileUtils.mkdir_p @seed_path
+          if File.exists? File.join(@seed_path, 'HEAD')
+            Dir.chdir @seed_path do
+              logger.info "Updating seed in #{@seed_path}"
+              # Need to be specific here as GH for windows adds an invalid "upstream" remote to all repos
+              Hem::Helper.shell 'git', 'fetch', 'origin'
+            end
+          else
+            logger.info "Cloning seed from #{@url} to #{@seed_path}"
+            Hem::Helper.shell 'git', 'clone', @url, @seed_path, '--mirror'
+          end
+        end
+
+        def vm_ip
+          [
+            10,
+            10,
+            [*0..255].sample,
+            [*2..255].sample
+          ].join('.')
+        end
+
+        def version
+          Dir.chdir @seed_path do
+            Hem::Helper.shell 'git', 'rev-parse', '--short', 'HEAD', :capture => true
+          end
+        end
+
+        class << self
+          def name_to_url name, options = {}
+            options = {
+              :use_short_seed_name => true
+            }.merge(options)
+
+            path = File.expand_path name
+            if name.include?(':')
+              name
+            elsif !options[:use_short_seed_name] || (name.match(/^(\.|\/|~)/) && path)
+              path
+            else
+              "git@github.com:inviqa/hem-seed-#{name}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hem/lib/self_signed_cert_generator.rb

@@ -0,0 +1,38 @@
+module Hem
+  module Lib
+    class SelfSignedCertGenerator
+      def self.generate domain
+        require 'openssl'
+        key = OpenSSL::PKey::RSA.new(2048)
+        public_key = key.public_key
+
+        subject = "/C=UK/ST=/O=Inviqa/localityName=London/commonName=*.#{domain}/organizationalUnitName=/emailAddress=support@inviqa.com"
+
+        cert = OpenSSL::X509::Certificate.new
+        cert.subject = cert.issuer = OpenSSL::X509::Name.parse(subject)
+        cert.not_before = Time.now
+        cert.not_after = Time.now + 365 * 24 * 60 * 60
+        cert.public_key = public_key
+        cert.serial = 0x0
+        cert.version = 2
+
+        ef = OpenSSL::X509::ExtensionFactory.new
+        ef.subject_certificate = cert
+        ef.issuer_certificate = cert
+        cert.extensions = [
+          ef.create_extension("basicConstraints","CA:TRUE", true),
+          ef.create_extension("subjectKeyIdentifier", "hash")
+        ]
+
+        cert.add_extension ef.create_extension("authorityKeyIdentifier", "keyid:always,issuer:always")
+
+        cert.sign key, OpenSSL::Digest::SHA1.new
+
+        {
+          :key => key.to_pem.to_s.strip,
+          :cert => cert.to_pem.to_s.strip
+        }
+      end
+    end
+  end
+end