dockly 1.5.8 → 1.5.9

data/lib/dockly/docker/registry.rb

@@ -7,7 +7,7 @@ class Dockly::Docker::Registry
   logger_prefix '[dockly docker registry]'
 
   dsl_attribute :name, :server_address, :email, :username, :password,
-                :authentication_required
+                :authentication_required, :auth_config_file
 
   default_value :server_address, DEFAULT_SERVER_ADDRESS
   default_value :authentication_required, true

data/lib/dockly/rake_task.rb

@@ -1,8 +1,6 @@
 require 'rake'
 require 'dockly'
 
-$rake_task_logger = Dockly::Util::Logger.new('[dockly rake_task]', STDOUT, false)
-
 class Rake::DebTask < Rake::Task
   def needed?
     raise "Package does not exist" if package.nil?
@@ -14,10 +12,25 @@ class Rake::DebTask < Rake::Task
   end
 end
 
+class Rake::DockerTask < Rake::Task
+  def needed?
+    raise "Docker does not exist" if docker.nil?
+    !docker.exists?
+  end
+
+  def docker
+    Dockly::Docker[name.split(':').last.to_sym]
+  end
+end
+
 module Rake::DSL
   def deb(*args, &block)
     Rake::DebTask.define_task(*args, &block)
   end
+
+  def docker(*args, &block)
+    Rake::DockerTask.define_task(*args, &block)
+  end
 end
 
 namespace :dockly do
@@ -33,4 +46,20 @@ namespace :dockly do
       end
     end
   end
+
+  namespace :docker do
+    Dockly.dockers.values.each do |inst|
+      docker inst.name => 'dockly:load' do
+        Thread.current[:rake_task] = inst.name
+        inst.generate!
+      end
+
+      namespace :noexport do
+        task inst.name => 'dockly:load' do
+          Thread.current[:rake_task] = inst.name
+          inst.generate_build
+        end
+      end
+    end
+  end
 end

data/lib/dockly/tar_diff.rb

@@ -0,0 +1,150 @@
+class Dockly::TarDiff
+  include Dockly::Util::Logger::Mixin
+
+  # Tar header format for a ustar tar
+  HEADER_UNPACK_FORMAT  = "Z100A8A8A8A12A12A8aZ100A6A2Z32Z32A8A8Z155"
+  PAX_FILE_FORMAT_REGEX = /\d+ path=(.*)/
+
+  logger_prefix '[dockly tar_diff]'
+
+  attr_reader :base, :output, :target, :base_enum, :target_enum
+
+  def initialize(base, target, output)
+    @base, @target, @output = base, target, output
+
+    @base_enum = to_enum(:read_header, base)
+    @target_enum = to_enum(:read_header, target)
+  end
+
+  def write_tar_section(header, data, remainder)
+    output.write(header)
+    output.write(data)
+    output.write("\0" * remainder)
+  end
+
+  def quick_write(size)
+    while size > 0
+      bread = target.read([size, 4096].min)
+      output.write(bread)
+      size -= bread.to_s.size
+    end
+  end
+
+  def read_header(io)
+    loop do
+      return if io.eof?
+      # Tar header is 512 bytes large
+      data = io.read(512)
+      fields = data.unpack(HEADER_UNPACK_FORMAT)
+      name = fields[0]
+      size = fields[4].oct
+      mtime = fields[5].oct
+      typeflag = fields[7]
+      prefix = fields[15]
+
+      empty = (data == "\0" * 512)
+      remainder = (512 - (size % 512)) % 512
+
+      yield data, name, prefix, mtime, typeflag, size, remainder, empty
+
+      io.read(remainder)
+    end
+  end
+
+  def process
+    debug "Started processing tar diff"
+    target_data = nil
+    base_data = nil
+    loop do
+      begin
+
+        target_header, target_name,  \
+        target_prefix, target_mtime,  \
+        target_typeflag,               \
+        target_size, target_remainder,  \
+        target_empty                     = target_enum.peek
+      rescue StopIteration
+        debug "Finished target file"
+        break
+      end
+
+      if target_empty
+        debug "End of target file/Empty"
+        break
+      end
+
+      begin
+        _, base_name, base_prefix, base_mtime, base_typeflag, base_size, _, base_empty = base_enum.peek
+      rescue StopIteration
+        target_data ||= target.read(target_size)
+        write_tar_section(target_header, target_data, target_remainder)
+        target_data = nil
+        target_enum.next
+        next
+      end
+
+      if base_empty
+        target_data ||= target.read(target_size)
+        write_tar_section(target_header, target_data, target_remainder)
+        target_data = nil
+        target_enum.next
+        next
+      end
+
+      target_full_name = File.join(target_prefix, target_name)
+      base_full_name = File.join(base_prefix, base_name)
+
+      target_full_name = target_full_name[1..-1] if target_full_name[0] == '/'
+      base_full_name = base_full_name[1..-1] if base_full_name[0] == '/'
+
+      if target_typeflag == 'x'
+        target_file = File.basename(target_full_name)
+        target_dir  = File.dirname(File.dirname(target_full_name))
+        target_full_name = File.join(target_dir, target_file)
+      end
+
+      if base_typeflag == 'x'
+        base_file = File.basename(base_full_name)
+        base_dir  = File.dirname(File.dirname(base_full_name))
+        base_full_name = File.join(base_dir, base_file)
+      end
+
+      # Remove the PaxHeader.PID from the file
+      # Format: /base/directory/PaxHeader.1234/file.ext
+      # After:  /base/directory/file.ext
+      if (target_typeflag == 'x' && base_typeflag == 'x')
+        target_data = target.read(target_size)
+        base_data = base.read(base_size)
+
+        if target_match = target_data.match(PAX_FILE_FORMAT_REGEX) && \
+            base_match = base_data.match(PAX_FILE_FORMAT_REGEX)
+          target_full_name = target_match[1]
+          base_full_name   = base_match[1]
+        end
+      end
+
+      if (target_full_name < base_full_name)
+        target_data ||= target.read(target_size)
+        write_tar_section(target_header, target_data, target_remainder)
+        target_data = nil
+        target_enum.next
+      elsif (base_full_name < target_full_name)
+        base.read(base_size) unless base_data
+        base_data = nil
+        base_enum.next
+      elsif (target_mtime != base_mtime) || (target_size != base_size)
+        target_data ||= target.read(target_size)
+        write_tar_section(target_header, target_data, target_remainder)
+        target_data = nil
+        target_enum.next
+      else
+        target.read(target_size) unless target_data
+        target_data = nil
+        target_enum.next
+        base.read(base_size) unless base_data
+        base_data = nil
+        base_enum.next
+      end
+    end
+  end
+end

data/lib/dockly/version.rb

@@ -1,3 +1,3 @@
 module Dockly
-  VERSION = '1.5.8'
+  VERSION = '1.5.9'
 end

data/snippets/docker_import.erb

@@ -0,0 +1 @@
+docker import - <% if data[:repo] %><%= data[:repo] %>:<%= data[:tag] %><% end %>

data/snippets/file_diff_docker_import.erb

@@ -0,0 +1,4 @@
+file_diff_docker_import_fn() {
+<%= get_from_s3(data[:base_image]) %>
+}
+(cat "<%= data[:diff_image] %>"; file_diff_docker_import_fn) | gunzip -c | <%= docker_import(data[:repo], data[:tag]) %>

data/snippets/file_docker_import.erb

@@ -0,0 +1 @@
+cat <%= data[:path] %> | gunzip -c | <%= docker_import(data[:repo], data[:tag]) %>

data/snippets/get_and_install_deb.erb

@@ -0,0 +1,2 @@
+<%= get_from_s3(data[:s3_url], data[:deb_path]) %>
+<%= install_package(data[:deb_path]) %>

data/snippets/get_from_s3.erb

@@ -0,0 +1,11 @@
+log "fetch: starting to fetch deb"
+worked=1
+s3_path="<%= data[:s3_url] %>"
+output_path="<%= data[:output_path] %>"
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  log "fetch: attempt ${attempt} to get $s3_path ..."
+  s3cmd -f get $s3_path $output_path && worked=0 || (log "fetch: attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "fetch: failed to pull deb from S3"
+log "fetch: successfully fetched deb"

data/snippets/install_package.erb

@@ -0,0 +1 @@
+dpkg -i "<%= data[:path] %>"

data/snippets/normalize_for_dockly.erb

@@ -0,0 +1,12 @@
+set -e -o pipefail
+
+function log {
+  logger -t dockly:startup "$@"
+}
+
+function fatal {
+  logger -t dockly:startup "FATAL: $@"
+  exit 1
+}
+
+mkdir -p /opt/dockly

data/snippets/registry_import.erb

@@ -0,0 +1 @@
+docker pull <%= data[:repo] %>:<%= data[:tag] %>

data/snippets/s3_diff_docker_import.erb

@@ -0,0 +1,14 @@
+<%
+base_image = "/opt/dockly/base_image.tar"
+%>
+
+s3_diff_docker_import_base_fn() {
+<%= get_from_s3(data[:base_image]) %>
+}
+s3_diff_docker_import_diff_fn() {
+<%= get_from_s3(data[:diff_image]) %>
+}
+s3_diff_docker_import_base_fn | gunzip -c > "<%= base_image %>"
+size=$(stat --format "%s" "<%= base_image %>")
+head_size=$(($size - 1024))
+(head -c $head_size "<%= base_image %>"; s3_diff_docker_import_diff_fn | gunzip -c) | <%= docker_import(data[:repo], data[:tag]) %>

data/snippets/s3_docker_import.erb

@@ -0,0 +1,4 @@
+s3_docker_import_fn() {
+<%= get_from_s3(data[:s3_url]) %>
+}
+s3_docker_import_fn | gunzip -c | <%= docker_import(data[:repo], data[:tag]) %>

data/spec/dockly/aws/s3_writer_spec.rb

@@ -0,0 +1,154 @@
+require 'spec_helper'
+
+describe Dockly::AWS::S3Writer do
+  let(:connection) { double(:connection) }
+  let(:bucket) { 'test_bucket' }
+  let(:object) { 'object_name.tar' }
+  let(:initiate_response) { double(:initiate_response) }
+  let(:upload_id) { 'test_id' }
+
+  subject { described_class.new(connection, bucket, object) }
+
+  before do
+    connection.should_receive(:initiate_multipart_upload) { initiate_response }
+    initiate_response.stub(:body) { { 'UploadId' => upload_id } }
+  end
+
+  describe ".new" do
+
+    it "sets the connection, s3_bucket, s3_object, and upload_id" do
+      expect(subject.connection).to eq(connection)
+      expect(subject.s3_bucket).to eq(bucket)
+      expect(subject.s3_object).to eq(object)
+      expect(subject.upload_id).to eq(upload_id)
+    end
+  end
+
+  describe "#upload_buffer" do
+    let(:message) { "message" }
+    let(:upload_response) { double(:upload_response) }
+    let(:etag) { "test" }
+
+    before do
+      connection.should_receive(:upload_part).with(bucket, object, upload_id, 1, message) do
+        upload_response
+      end
+      upload_response.stub(:headers) { { "ETag" => etag } }
+      subject.instance_variable_set(:"@buffer", message)
+    end
+
+    it "connects to S3" do
+      subject.upload_buffer
+      expect(subject.instance_variable_get(:"@parts")).to include(etag)
+    end
+  end
+
+  describe "#write" do
+    let(:message) { "a" * chunk_length }
+
+    context "with a buffer of less than 5 MB" do
+      let(:chunk_length) { 100 }
+
+      before do
+        subject.should_not_receive(:upload_buffer)
+      end
+
+      it "adds it to the buffer and returns the chunk length" do
+        expect(subject.write(message)).to eq(chunk_length)
+        expect(subject.instance_variable_get(:"@buffer")).to eq(message)
+      end
+    end
+
+    context "with a buffer of greater than 5 MB"  do
+      let(:chunk_length) { 1 + 5 * 1024 * 1024 }
+
+      before do
+        subject.should_receive(:upload_buffer)
+      end
+
+      it "adds it to the buffer, writes to S3 and returns the chunk length" do
+        expect(subject.write(message)).to eq(chunk_length)
+      end
+    end
+  end
+
+  describe "#close" do
+    let(:complete_response) { double(:complete_response) }
+
+    before do
+      connection.should_receive(:complete_multipart_upload).with(bucket, object, upload_id, []) do
+        complete_response
+      end
+    end
+
+    context "when it passes" do
+      before do
+        complete_response.stub(:body) { {} }
+      end
+
+      context "when the buffer is not empty" do
+        before do
+          subject.instance_variable_set(:"@buffer", "text")
+          subject.should_receive(:upload_buffer)
+        end
+
+        it "uploads the rest of the buffer and closes the connection" do
+          expect(subject.close).to be_true
+        end
+      end
+
+      context "when the buffer is empty" do
+        before do
+          subject.should_not_receive(:upload_buffer)
+        end
+
+        it "closes the connection" do
+          expect(subject.close).to be_true
+        end
+      end
+    end
+
+    context "when it fails" do
+      before do
+        complete_response.stub(:body) { { 'Code' => 20, 'Message' => 'Msggg' } }
+      end
+
+      it "raises an error" do
+        expect { subject.close }.to raise_error("Failed to upload to S3: 20: Msggg")
+      end
+    end
+  end
+
+  describe "#abort" do
+    before do
+      connection.should_receive(:abort_multipart_upload).with(bucket, object, upload_id)
+    end
+
+    it "aborts the upload" do
+      subject.abort
+    end
+  end
+
+  describe "#abort_unless_closed" do
+    context "when the upload is closed" do
+      before do
+        subject.should_not_receive(:abort)
+        subject.instance_variable_set(:"@closed", true)
+      end
+
+      it "does not abort" do
+        subject.abort_unless_closed
+      end
+    end
+
+    context "when the upload is open" do
+      before do
+        subject.should_receive(:abort)
+      end
+
+      it "aborts the upload" do
+        subject.abort_unless_closed
+      end
+    end
+  end
+end