dockly 2.7.2 → 3.0.0

data/lib/dockly/s3_writer.rb

@@ -0,0 +1,79 @@
+module Dockly
+  class S3Writer
+    include Dockly::Util::Logger::Mixin
+    extend Forwardable
+
+    MAX_BUFFER_SIZE = 5 * 1024 * 1024
+
+    attr_reader :connection, :s3_bucket, :s3_object, :parts, :closed, :buffer
+
+    def_delegators :multipart_upload, :upload_id
+    logger_prefix '[dockly s3writer]'
+
+    def initialize(connection, s3_bucket, s3_object)
+      @connection = connection
+      @s3_bucket = s3_bucket
+      @s3_object = s3_object
+      @parts = []
+      @closed = false
+      @buffer = StringIO.new
+    end
+
+    def upload_buffer
+      num = @parts.length.succ
+      debug "Writing chunk ##{num} to s3://#{s3_bucket}/#{s3_object} with upload id: #{upload_id}"
+      res = connection.upload_part(
+        bucket: s3_bucket,
+        key: s3_object,
+        upload_id: upload_id,
+        part_number:num,
+        body: buffer.tap(&:rewind)
+      )
+      @parts << res.etag
+      @buffer = StringIO.new
+    end
+
+    def write(chunk)
+      @buffer.write(chunk)
+      upload_buffer if buffer.size > MAX_BUFFER_SIZE
+      chunk.length
+    end
+
+    def close
+      return if @closed
+      upload_buffer unless buffer.size.zero?
+      connection.complete_multipart_upload(
+        bucket: s3_bucket,
+        key: s3_object,
+        upload_id: upload_id,
+        multipart_upload: {
+          parts: @parts.each_with_index.map do |part, idx|
+            {
+              etag: part,
+              part_number: idx.succ
+            }
+          end
+        }
+      )
+      @closed = true
+    end
+
+    def abort_upload
+      connection.abort_multipart_upload(
+        bucket: s3_bucket,
+        key: s3_object,
+        upload_id: upload_id
+      )
+    end
+
+    def abort_unless_closed
+      abort_upload unless @closed
+      @closed = true
+    end
+
+    def multipart_upload
+      @multipart_upload ||=
+        connection.create_multipart_upload(bucket: s3_bucket, key: s3_object)
+    end
+  end
+end

data/lib/dockly/util/git.rb

@@ -1,13 +1,38 @@
-require 'grit'
-
 module Dockly::Util::Git
-  extend self
+  module_function
+
+  def repo
+    @repo ||= Rugged::Repository.discover('.')
+  end
+
+  def sha
+    return @sha if @sha
+    @sha = repo.head.target.oid[0..6]
+  rescue
+    @sha = 'unknown'
+  end
 
-  def git_repo
-    @git_repo ||= Grit::Repo.new('.')
+  def ls_files(oid)
+    target = repo.lookup(oid)
+    target = target.target until target.type == :commit
+    ary = []
+    target.tree.walk(:postorder) do |root, entry|
+      next unless entry[:type] == :blob
+      name = File.join(root, entry[:name]).gsub(/\A\//, '')
+      ary << entry.merge(name: name)
+    end
+    ary
   end
 
-  def git_sha
-    @git_sha ||= git_repo.git.show.lines.first.chomp.match(/^commit ([a-f0-9]+)$/)[1][0..6] rescue 'unknown'
+  def archive(oid, prefix, output)
+    Gem::Package::TarWriter.new(output) do |tar|
+      ls_files(oid).each do |blob|
+        name, mode = blob.values_at(:name, :filemode)
+        prefixed = File.join(prefix, name)
+        tar.add_file(prefixed, mode) do |tar_out|
+          tar_out.write(File.read(name))
+        end
+      end
+    end
   end
 end

data/lib/dockly/version.rb

@@ -1,3 +1,8 @@
 module Dockly
-  VERSION = '2.7.2'
+  MAJOR = 3
+  MINOR = 0
+  PATCH = 0
+  RELEASE = nil
+
+  VERSION = [MAJOR, MINOR, PATCH, RELEASE].compact.join('.')
 end

data/spec/dockly/build_cache/base_spec.rb

@@ -13,15 +13,19 @@ describe Dockly::BuildCache::Base do
 
   describe '#up_to_date?' do
     context 'when the object exists in s3' do
-      before { subject.connection.stub(:head_object) }
+      before do
+        allow(subject.connection)
+          .to receive(:head_object)
+      end
 
       its(:up_to_date?) { should be_true }
     end
 
     context 'when the object does not exist in s3' do
       before do
-        subject.connection.stub(:head_object)
-            .and_raise(Excon::Errors::NotFound.new('help'))
+        allow(subject.connection)
+          .to receive(:head_object)
+          .and_raise(Aws::S3::Errors::NoSuchKey.new('Some Error', 500))
       end
 
       its(:up_to_date?) { should be_false }
@@ -33,8 +37,12 @@ describe Dockly::BuildCache::Base do
     let(:object) { double(:object) }
 
     before do
-      subject.connection.stub(:get_object).and_return object
-      object.stub(:body).and_return 'hey dad'
+      allow(subject.connection)
+        .to receive(:get_object)
+        .and_return(object)
+      allow(object)
+        .to receive(:body)
+        .and_return(StringIO.new('hey dad').tap(&:rewind))
     end
 
     after do
@@ -50,8 +58,8 @@ describe Dockly::BuildCache::Base do
 
   describe '#s3_object' do
     before do
-      subject.stub(:s3_object_prefix) { 'lol' }
-      subject.stub(:hash_output) { 'lel' }
+      allow(subject).to receive(:s3_object_prefix).and_return('lol')
+      allow(subject).to receive(:hash_output).and_return('lel')
     end
 
     context "without an arch_output" do

data/spec/dockly/build_cache/local_spec.rb

@@ -83,7 +83,7 @@ describe Dockly::BuildCache::Local do
 
   describe '#hash_output' do
     let(:output) {
-      "5ebe2fbf321fa1008833574649e986a3"
+      "f683463a09482287c33959ab71a87189"
     }
 
     context "when hash command returns successfully" do

data/spec/dockly/deb_spec.rb

@@ -191,7 +191,9 @@ describe Dockly::Deb do
 
     context 'when the object does exist' do
       before do
-        Dockly::AWS.s3.stub(:head_object).and_return {}
+        allow(Dockly.s3)
+          .to receive(:head_object)
+          .and_return({})
       end
 
       it 'is true' do
@@ -201,7 +203,9 @@ describe Dockly::Deb do
 
     context 'when the object does not exist' do
       before do
-        Dockly::AWS.s3.stub(:head_object).and_raise(Excon::Errors::NotFound.new "NotFound")
+        allow(Dockly.s3)
+          .to receive(:head_object)
+          .and_raise(StandardError.new('object does not exist'))
       end
 
       it 'is true' do
@@ -224,7 +228,7 @@ describe Dockly::Deb do
 
     context 'when the s3_bucket is nil' do
       it 'does nothing' do
-        Dockly::AWS.should_not_receive(:s3)
+        expect(Dockly).to_not receive(:s3)
         subject.upload_to_s3
       end
     end
@@ -244,14 +248,14 @@ describe Dockly::Deb do
       context 'when the package has been created' do
         before { subject.create_package! }
 
-        it 'creates the s3 bucket' do
-          subject.upload_to_s3
-          Dockly::AWS.s3.get_bucket(bucket_name).body.should_not be_nil
-        end
-
         it 'inserts the deb package into that bucket' do
+          expect(Dockly.s3).to receive(:put_object) do |hash|
+            expect(hash[:bucket]).to eq(bucket_name)
+            expect(hash[:key]).to eq(subject.s3_object_name)
+            expect(hash).to have_key(:body)
+          end
+
           subject.upload_to_s3
-          Dockly::AWS.s3.get_bucket(bucket_name, subject.s3_object_name).body.should_not be_nil
         end
       end
     end

data/spec/dockly/docker_spec.rb

@@ -135,8 +135,10 @@ describe Dockly::Docker do
         let(:data) { 'sweet, sweet data' }
 
         before do
-          subject.send(:connection).put_bucket('bucket')
-          subject.send(:connection).put_object('bucket', 'object', data)
+          allow(Dockly.s3)
+            .to receive(:get_object)
+            .with(bucket: 'bucket', key: 'object')
+            .and_yield(data)
         end
 
         it 'pulls the file from S3' do
@@ -182,7 +184,7 @@ describe Dockly::Docker do
     context "with an S3 export" do
       let(:export) { double(:export) }
       before do
-        expect(Dockly::AWS::S3Writer).to receive(:new).and_return(export)
+        expect(Dockly::S3Writer).to receive(:new).and_return(export)
         expect(export).to receive(:write).once
         expect(export).to receive(:close).once
         subject.s3_bucket "test-bucket"

data/spec/dockly/history_spec.rb

@@ -68,7 +68,7 @@ describe Dockly::History do
       expect(files).to be_a(Array)
       expect(files).to include('dockly.gemspec')
       expect(files).to include('lib/dockly.rb')
-      expect(files).to include('spec/dockly/aws/s3_writer_spec.rb')
+      expect(files).to include('spec/dockly/s3_writer_spec.rb')
     end
   end
 

data/spec/dockly/rpm_spec.rb

@@ -178,7 +178,9 @@ describe Dockly::Rpm do
 
     context 'when the object does exist' do
       before do
-        Dockly::AWS.s3.stub(:head_object).and_return {}
+        allow(Dockly.s3)
+          .to receive(:head_object)
+          .and_return({})
       end
 
       it 'is true' do
@@ -188,7 +190,9 @@ describe Dockly::Rpm do
 
     context 'when the object does not exist' do
       before do
-        Dockly::AWS.s3.stub(:head_object).and_raise(Excon::Errors::NotFound.new "NotFound")
+        allow(Dockly.s3)
+          .to receive(:head_object)
+          .and_raise(StandardError.new('object does not exist'))
       end
 
       it 'is true' do
@@ -211,7 +215,7 @@ describe Dockly::Rpm do
 
     context 'when the s3_bucket is nil' do
       it 'does nothing' do
-        Dockly::AWS.should_not_receive(:s3)
+        expect(Dockly).to_not receive(:s3)
         subject.upload_to_s3
       end
     end
@@ -231,14 +235,14 @@ describe Dockly::Rpm do
       context 'when the package has been created' do
         before { subject.create_package! }
 
-        it 'creates the s3 bucket' do
-          subject.upload_to_s3
-          Dockly::AWS.s3.get_bucket(bucket_name).body.should_not be_nil
-        end
-
         it 'inserts the rpm package into that bucket' do
+          expect(Dockly.s3).to receive(:put_object) do |hash|
+            expect(hash[:bucket]).to eq(bucket_name)
+            expect(hash[:key]).to eq(subject.s3_object_name)
+            expect(hash).to have_key(:body)
+          end
+
           subject.upload_to_s3
-          Dockly::AWS.s3.get_bucket(bucket_name, subject.s3_object_name).body.should_not be_nil
         end
       end
     end

data/spec/dockly/s3_writer_spec.rb

@@ -0,0 +1,139 @@
+require 'spec_helper'
+
+describe Dockly::S3Writer do
+  let(:connection) { double(:connection) }
+  let(:bucket) { 'test_bucket' }
+  let(:object) { 'object_name.tar' }
+  let(:multipart_upload) { double(:multipart_upload, upload_id: upload_id) }
+  let(:upload_id) { 'test_id' }
+
+  before do
+    allow(subject)
+      .to receive(:multipart_upload)
+      .and_return(multipart_upload)
+  end
+
+  subject { described_class.new(connection, bucket, object) }
+
+  describe '.new' do
+    it 'sets the connection, s3_bucket, s3_object, and upload_id' do
+      expect(subject.connection).to eq(connection)
+      expect(subject.s3_bucket).to eq(bucket)
+      expect(subject.s3_object).to eq(object)
+    end
+  end
+
+  describe '#upload_id' do
+    it 'delegates to the multipart_upload' do
+      expect(subject.upload_id).to eq(multipart_upload.upload_id)
+    end
+  end
+
+  describe '#upload_buffer' do
+    let(:input) { 'Some String' }
+    let(:io) { StringIO.new(input) }
+    let(:upload_response) { double(:upload_response, etag: etag) }
+    let(:etag) { 'test' }
+
+    before do
+      subject.instance_variable_set(:@buffer, io)
+
+      allow(connection)
+        .to receive(:upload_part)
+        .with(bucket: bucket, key: object, upload_id: upload_id, part_number: 1, body: io)
+        .and_return(upload_response)
+    end
+
+    it 'uploads to S3' do
+      expect { subject.upload_buffer }
+        .to change { subject.parts.last }
+        .to(etag)
+    end
+
+    it 'clears the buffer' do
+      expect { subject.upload_buffer }
+        .to change { subject.buffer.tap(&:rewind).string }
+        .from(input)
+        .to('')
+    end
+  end
+
+  describe '#write' do
+    let(:message) { 'a' * chunk_length }
+
+    context 'with a buffer of less than 5 MB' do
+      let(:chunk_length) { 100 }
+
+      it 'adds it to the buffer and returns the chunk length' do
+        expect(subject).to_not receive(:upload_buffer)
+        expect(subject.write(message)).to eq(chunk_length)
+        expect(subject.buffer.tap(&:rewind).string).to eq(message)
+      end
+    end
+
+    context 'with a buffer of greater than 5 MB'  do
+      let(:chunk_length) { 1 + 5 * 1024 * 1024 }
+
+      it 'adds it to the buffer, writes to S3 and returns the chunk length' do
+        expect(subject).to receive(:upload_buffer)
+        expect(subject.write(message)).to eq(chunk_length)
+      end
+    end
+  end
+
+  describe '#close' do
+    let(:complete_response) { double(:complete_response) }
+
+    before do
+      allow(connection)
+        .to receive(:complete_multipart_upload)
+        .with(bucket: bucket, key: object, upload_id: upload_id, multipart_upload: { parts: [] })
+        .and_return(complete_response)
+    end
+
+    context 'when it passes' do
+      context 'when the buffer is not empty' do
+        before { subject.instance_variable_set(:@buffer, StringIO.new('text')) }
+
+        it 'uploads the rest of the buffer and closes the connection' do
+          expect(subject).to receive(:upload_buffer)
+          expect(subject.close).to be_true
+        end
+      end
+
+      context 'when the buffer is empty' do
+        it 'closes the connection' do
+          expect(subject).to_not receive(:upload_buffer)
+          expect(subject.close).to be_true
+        end
+      end
+    end
+  end
+
+  describe '#abort_upload' do
+    it 'aborts the upload' do
+      expect(connection)
+        .to receive(:abort_multipart_upload)
+        .with(bucket: bucket, key: object, upload_id: upload_id)
+      subject.abort_upload
+    end
+  end
+
+  describe '#abort_unless_closed' do
+    context 'when the upload is closed' do
+      before { subject.instance_variable_set(:@closed, true) }
+
+      it 'does not abort' do
+        expect(subject).to_not receive(:abort_upload)
+        subject.abort_unless_closed
+      end
+    end
+
+    context 'when the upload is open' do
+      it 'aborts the upload' do
+        expect(subject).to receive(:abort_upload)
+        subject.abort_unless_closed
+      end
+    end
+  end
+end