dockly 1.12.3 → 1.13.0

data/.travis.yml

@@ -1,8 +1,12 @@
 language: ruby
 rvm:
-    - 1.9.3
-    - 2.0.0
-script: CI=true bundle exec rake
+    - 1.9
+    - 2.0
+    - 2.1
+    - 2.2
+script:
+  - git fetch --tags
+  - CI=true bundle exec rake
 before_install:
  - sudo apt-get update -qq
  - sudo apt-get install -qq rpm

data/Rakefile

@@ -6,6 +6,11 @@ require 'rake'
 require 'dockly'
 require 'rspec/core/rake_task'
 require 'cane/rake_task'
+require 'pry'
+
+task :shell do
+  Pry.start(Dockly)
+end
 
 task :default => [:spec, :quality]
 

data/dockly.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'foreman'
   gem.add_dependency 'fpm', '~> 1.2.0'
   gem.add_dependency 'grit'
+  gem.add_dependency 'rugged'
   gem.add_development_dependency 'cane'
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rake'

data/lib/dockly.rb

@@ -4,6 +4,7 @@ require 'dockly/util/git'
 require 'fog'
 require 'foreman/cli_fix'
 require 'foreman/export/base_fix'
+require 'rugged'
 
 module Dockly
   attr_reader :instance, :git_sha
@@ -15,6 +16,7 @@ module Dockly
   autoload :BuildCache, 'dockly/build_cache'
   autoload :Docker, 'dockly/docker'
   autoload :Deb, 'dockly/deb'
+  autoload :History, 'dockly/history'
   autoload :Rpm, 'dockly/rpm'
   autoload :TarDiff, 'dockly/tar_diff'
 

data/lib/dockly/deb.rb

@@ -48,6 +48,14 @@ class Dockly::Deb
     upload_to_s3
   end
 
+  def copy_from_s3(sha)
+    ensure_present! :s3_bucket
+    object = s3_object_name_for(sha)
+    info "Copying s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object_name}"
+    Dockly::AWS.s3.copy_object(s3_bucket, object, s3_bucket, s3_object_name)
+    info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object_name}"
+  end
+
   def build_path
     ensure_present! :build_dir, :deb_build_dir
     File.join(build_dir, deb_build_dir, output_filename)
@@ -59,8 +67,7 @@ class Dockly::Deb
     info "#{name}: found package: #{s3_url}"
     true
   rescue
-    info "#{name}: could not find package: " +
-         "#{s3_url}"
+    info "#{name}: could not find package: #{s3_url}"
     false
   end
 
@@ -78,7 +85,11 @@ class Dockly::Deb
   end
 
   def s3_object_name
-    "#{package_name}/#{Dockly::Util::Git.git_sha}/#{output_filename}"
+    s3_object_name_for(Dockly::Util::Git.git_sha)
+  end
+
+  def s3_object_name_for(sha)
+    "#{package_name}/#{sha}/#{output_filename}"
   end
 
   def output_filename

data/lib/dockly/docker.rb

@@ -32,6 +32,14 @@ class Dockly::Docker
     (@build_env ||= {}).tap { |env| env.merge!(hash) if hash.is_a?(Hash) }
   end
 
+  def copy_from_s3(sha)
+    return if s3_bucket.nil?
+    object = s3_object_for(sha)
+    info "Copying s3://#{s3_bucket}/#{object} to #{s3_bucket}/#{s3_object}"
+    Dockly::AWS.s3.copy_object(s3_bucket, object, s3_bucket, s3_object)
+    info "Successfully copied s3://#{s3_bucket}/#{object} to s3://#{s3_bucket}/#{s3_object}"
+  end
+
   def generate!
     image = generate_build
     export_image(image)
@@ -220,10 +228,10 @@ class Dockly::Docker
       container = image.run('true')
       info "created the container: #{container.id}"
 
-      unless s3_bucket.nil?
-        output = Dockly::AWS::S3Writer.new(connection, s3_bucket, s3_object)
-      else
+      if s3_bucket.nil?
         output = File.open(tar_path, 'wb')
+      else
+        output = Dockly::AWS::S3Writer.new(connection, s3_bucket, s3_object)
       end
 
       gzip_output = Zlib::GzipWriter.new(output)
@@ -285,9 +293,11 @@ class Dockly::Docker
   end
 
   def s3_object
-    output = "#{s3_object_prefix}"
-    output << "#{Dockly::Util::Git.git_sha}/"
-    output << "#{export_filename}"
+    s3_object_for(Dockly::Util::Git.git_sha)
+  end
+
+  def s3_object_for(sha)
+    [s3_object_prefix, sha, '/', export_filename].join
   end
 
   def push_to_registry(image)

data/lib/dockly/history.rb

@@ -0,0 +1,64 @@
+# This module contains logic to find matching content hash for a given commit.
+module Dockly::History
+  module_function
+
+  ASCII_FILE_SEP = 28.chr
+  ASCII_RECORD_SEP = 30.chr
+
+  TAG_PREFIX = 'dockly-'
+
+  def push_content_tag!
+    fail 'An SSH agent must be running to push the tag' if ENV['SSH_AGENT_PID'].nil?
+    refs = ["refs/tags/#{content_tag}"]
+    repo.remotes.each do |remote|
+      username = remote.url.split('@').first
+      creds = Rugged::Credentials::SshKeyFromAgent.new(username: username)
+      remote.push(refs, credentials: creds)
+    end
+  end
+
+  def write_content_tag!
+    repo.tags.create(content_tag, repo.head.target_id, true)
+  end
+
+  def duplicate_build?
+    !duplicate_build_sha.nil?
+  end
+
+  def duplicate_build_sha
+    return @duplicate_build_sha if @duplicate_build_sha
+    sha = tags[content_tag]
+    @duplicate_build_sha = sha unless sha == repo.head.target_id
+  end
+
+  def tags
+    @tags ||= Hash.new do |hash, key|
+      tag = repo.tags[key]
+      hash[key] = tag.target_id if tag
+    end
+  end
+
+  def content_tag
+    @content_tag ||= TAG_PREFIX + content_hash_for(ls_files)
+  end
+
+  def ls_files
+    repo.head.target.tree.walk(:preorder)
+      .map { |root, elem| [root, elem[:name]].compact.join }
+      .select(&File.method(:file?))
+  end
+
+  def content_hash_for(paths)
+    paths.sort.each_with_object(Digest::SHA384.new) do |path, hash|
+      next unless File.exist?(path)
+      mode = File::Stat.new(path).mode
+      data = File.read(path)
+      str = [path, mode, data].join(ASCII_RECORD_SEP.chr) + ASCII_FILE_SEP.chr
+      hash.update(str)
+    end.hexdigest
+  end
+
+  def repo
+    @repo ||= Rugged::Repository.discover('.')
+  end
+end

data/lib/dockly/rake_task.rb

@@ -56,6 +56,7 @@ namespace :dockly do
   prepare_targets = []
   upload_targets = []
   build_targets = []
+  copy_targets = []
 
   namespace :deb do
     Dockly.debs.values.each do |inst|
@@ -71,6 +72,12 @@ namespace :dockly do
         end
       end
 
+      namespace :copy do
+        task inst.name => 'dockly:load' do |name|
+          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
+        end
+      end
+
       deb inst.name => [
         'dockly:load',
         "dockly:deb:prepare:#{inst.name}",
@@ -78,6 +85,7 @@ namespace :dockly do
       ]
       prepare_targets << "dockly:deb:prepare:#{inst.name}"
       upload_targets << "dockly:deb:upload:#{inst.name}"
+      copy_targets << "dockly:deb:copy:#{inst.name}"
       build_targets << "dockly:deb:#{inst.name}"
     end
   end
@@ -96,6 +104,12 @@ namespace :dockly do
         end
       end
 
+      namespace :copy do
+        task inst.name => 'dockly:load' do |name|
+          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
+        end
+      end
+
       rpm inst.name => [
         'dockly:load',
         "dockly:rpm:prepare:#{inst.name}",
@@ -103,6 +117,7 @@ namespace :dockly do
       ]
       prepare_targets << "dockly:rpm:prepare:#{inst.name}"
       upload_targets << "dockly:rpm:upload:#{inst.name}"
+      copy_targets << "dockly:rpm:copy:#{inst.name}"
       build_targets << "dockly:rpm:#{inst.name}"
     end
   end
@@ -128,6 +143,13 @@ namespace :dockly do
         end
       end
 
+      namespace :copy do
+        task inst.name => 'dockly:load' do
+          Thread.current[:rake_task] = inst.name
+          inst.copy_from_s3(Dockly::History.duplicate_build_sha[0..6])
+        end
+      end
+
       docker inst.name => [
         'dockly:load',
         "dockly:docker:prepare:#{inst.name}",
@@ -138,6 +160,7 @@ namespace :dockly do
       unless inst.s3_bucket.nil?
         prepare_targets << "dockly:docker:prepare:#{inst.name}"
         upload_targets << "dockly:docker:upload:#{inst.name}"
+        copy_targets << "dockly:docker:copy:#{inst.name}"
         build_targets << "dockly:docker:#{inst.name}"
       end
     end
@@ -146,4 +169,15 @@ namespace :dockly do
   multitask :prepare_all => prepare_targets
   multitask :upload_all => upload_targets
   multitask :build_all => build_targets
+  multitask :copy_all => copy_targets
+
+  task :build_or_copy_all do
+    if Dockly::History.duplicate_build?
+      Rake::Task['dockly:copy_all'].invoke
+    else
+      Rake::Task['dockly:build_all'].invoke
+      Dockly::History.write_content_tag!
+      Dockly::History.push_content_tag!
+    end
+  end
 end

data/lib/dockly/version.rb

@@ -1,3 +1,3 @@
 module Dockly
-  VERSION = '1.12.3'
+  VERSION = '1.13.0'
 end

data/spec/dockly/history_spec.rb

@@ -0,0 +1,94 @@
+require 'spec_helper'
+
+describe Dockly::History do
+  describe '#duplicate_build?' do
+    before { allow(subject).to receive(:duplicate_build_sha).and_return(sha) }
+
+    context 'when the duplicate build sha is not present' do
+      let(:sha) { nil }
+
+      it 'returns false' do
+        expect(subject).to_not be_duplicate_build
+      end
+    end
+
+    context 'when the duplicate build sha is present' do
+      let(:sha) { 'DEADBEEF' }
+
+      it 'returns true' do
+        expect(subject).to be_duplicate_build
+      end
+    end
+  end
+
+  describe '#duplicate_build_sha' do
+    let(:content_tag) { 'dockly-FAKE-CONTENT-HASH' }
+
+    before { allow(subject).to receive(:content_tag).and_return(content_tag) }
+
+    context 'when there is not a commit with a matching content hash' do
+      it 'returns nil' do
+        expect(subject.duplicate_build_sha).to be(nil)
+      end
+    end
+
+    context 'when there is a commit with a matching content hash' do
+      let(:sha) { 'dockly-FAKE-SHA' }
+      let(:key) { content_tag }
+
+      before { subject.tags[key] = sha }
+      after { subject.tags.delete(key) }
+
+      it 'returns that commit' do
+        expect(subject.duplicate_build_sha).to eq(sha)
+      end
+    end
+  end
+
+  describe '#tags' do
+    let(:expected) do
+      {
+        'v1.4.0' => '708b4f1bd7846c258e2151e34f8f746b399ee6fb',
+        'v1.4.2' => '20b62fa084ac903a8753cfc7939c7e31c86761d2',
+        'v1.4.3' => '8eac23450975d91f0aa4c62c6985ec4f4ce7594f'
+      }
+    end
+
+    it 'returns a Hash of the git tags' do
+      expected.each do |tag, oid|
+        expect(subject.tags[tag]).to include(oid)
+      end
+    end
+  end
+
+  describe '#ls_files' do
+    let(:files) { subject.ls_files }
+
+    it 'returns the files checked into the repo' do
+      expect(files).to be_a(Array)
+      expect(files).to include('dockly.gemspec')
+      expect(files).to include('lib/dockly.rb')
+      expect(files).to include('spec/dockly/aws/s3_writer_spec.rb')
+    end
+  end
+
+  describe '#content_hash_for' do
+    context 'when two hashes are compared' do
+      let(:hash_one) { subject.content_hash_for(%w(dockly.gemspec Gemfile)) }
+      let(:hash_two) { subject.content_hash_for(%w(lib/dockly.rb LICENSE.txt)) }
+
+      it 'creates a unique hash for the given paths' do
+        expect(hash_one).to_not eq(hash_two)
+      end
+    end
+
+    context 'when the paths are in a different order' do
+      let(:hash_one) { subject.content_hash_for(%w(dockly.gemspec Gemfile)) }
+      let(:hash_two) { subject.content_hash_for(%w(Gemfile dockly.gemspec)) }
+
+      it 'still creates the same hash' do
+        expect(hash_one).to eq(hash_two)
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: dockly
 version: !ruby/object:Gem::Version
-  version: 1.12.3
+  version: 1.13.0
+  prerelease: 
 platform: ruby
 authors:
 - Swipely, Inc.
@@ -13,6 +14,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: clamp
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: dockly-util
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -55,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,6 +70,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -69,6 +78,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -76,6 +86,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -83,6 +94,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: foreman
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -90,6 +102,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -97,6 +110,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: fpm
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -104,6 +118,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -111,6 +126,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: grit
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -118,6 +134,23 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rugged
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -125,6 +158,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: cane
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -132,6 +166,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -139,6 +174,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -146,6 +182,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -153,6 +190,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -160,6 +198,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -167,6 +206,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -174,6 +214,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -181,6 +222,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -188,6 +230,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -195,6 +238,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -202,6 +246,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -241,6 +286,7 @@ files:
 - lib/dockly/docker.rb
 - lib/dockly/docker/registry.rb
 - lib/dockly/foreman.rb
+- lib/dockly/history.rb
 - lib/dockly/rake_task.rb
 - lib/dockly/rpm.rb
 - lib/dockly/tar_diff.rb
@@ -270,6 +316,7 @@ files:
 - spec/dockly/docker/registry_spec.rb
 - spec/dockly/docker_spec.rb
 - spec/dockly/foreman_spec.rb
+- spec/dockly/history_spec.rb
 - spec/dockly/rpm_spec.rb
 - spec/dockly/tar_diff_spec.rb
 - spec/dockly_spec.rb
@@ -285,26 +332,27 @@ files:
 homepage: https://github.com/swipely/dockly
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 1.8.23.2
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Packaging made easy
 test_files:
 - spec/dockly/aws/s3_writer_spec.rb
@@ -318,6 +366,7 @@ test_files:
 - spec/dockly/docker/registry_spec.rb
 - spec/dockly/docker_spec.rb
 - spec/dockly/foreman_spec.rb
+- spec/dockly/history_spec.rb
 - spec/dockly/rpm_spec.rb
 - spec/dockly/tar_diff_spec.rb
 - spec/dockly_spec.rb
@@ -330,3 +379,4 @@ test_files:
 - spec/fixtures/test-2.tar.gz
 - spec/fixtures/test-3.tar
 - spec/spec_helper.rb
+has_rdoc: 

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ZDVjYWM1NjRjYjgxMTJkMjYyYTE4NWNhYjY4Mjc0YTU0MzljNzNkMw==
-  data.tar.gz: !binary |-
-    MmFjYWY4Y2Q0NGQxMjhhMjgwZDlkNDk5MDUxYmMzYjFkMjM2OTcwMw==
-SHA512:
-  metadata.gz: !binary |-
-    NTJmNjhhOTAwNzJmMjdjZWEwN2MzMGMxMTNkOGE3M2Q2Njg4NzJhNWE0ODky
-    MmE4OWU0ODU3YjA4MGU1OTg5YTVkZDBmN2QyMTJhYWIwMGI2YTljZjMzOGJk
-    YWVlZjU4OTE4ZTY1ZjczMjg2ZmE3NzVjYzE2ZTk3MDczYzhhNWQ=
-  data.tar.gz: !binary |-
-    MTRiNzUxMThhYTlmMTA5NWVmMGVkODJhODliNDg4YWViZDU5NmYwZTQ4YmE5
-    OWM2OGIwZTJhODM0OWMzN2Q5ZWVhZjIwYTQwM2RiOWIyZmM3NWY3ZmM0YjM3
-    MzRlNmJhN2Q3YTFmNjNjOTc1MjUwMGMxMjk4NzljNjc4ZWU3MzA=