dockly 1.6.0 → 1.7.0

data/lib/dockly/bash_builder.rb

@@ -16,15 +16,14 @@ module Dockly
     end
 
     generate_snippet_for :normalize_for_dockly, []
-    generate_snippet_for :get_from_s3, [:s3_url, :output_path], { :output_path => "-" }
-    generate_snippet_for :install_package, [:path]
     generate_snippet_for :get_and_install_deb, [:s3_url, :deb_path]
-    generate_snippet_for :docker_import, [:repo, :tag], { :tag => "latest" }
-    generate_snippet_for :docker_tag_latest, [:repo, :tag]
+
     generate_snippet_for :file_docker_import, [:path, :repo, :tag]
     generate_snippet_for :file_diff_docker_import, [:base_image, :diff_image, :repo, :tag]
     generate_snippet_for :s3_docker_import, [:s3_url, :repo, :tag]
     generate_snippet_for :s3_diff_docker_import, [:base_image, :diff_image, :repo, :tag]
     generate_snippet_for :registry_import, [:repo, :tag], { :tag => "latest" }
+
+    generate_snippet_for :docker_tag_latest, [:repo, :tag]
   end
 end

data/lib/dockly/version.rb

@@ -1,3 +1,3 @@
 module Dockly
-  VERSION = '1.6.0'
+  VERSION = '1.7.0'
 end

data/snippets/file_diff_docker_import.erb

@@ -1,4 +1,20 @@
 file_diff_docker_import_fn() {
-<%= get_from_s3(data[:base_image]) %>
+  s3_path="<%= data[:base_image] %>"
+  log "fetch: starting to fetch $s3_path"
+  s3cmd -f get $s3_path - 2> >(log)
+  log "fetch: successfully fetched $s3_path"
 }
-(cat "<%= data[:diff_image] %>"; file_diff_docker_import_fn) | gunzip -c | <%= docker_import(data[:repo], data[:tag]) %>
+
+docker_import() {
+  repo=<%= data[:repo] %>
+  tag=<%= data[:tag] %>
+  (cat "<%= data[:diff_image] %>"; file_diff_docker_import_fn) | gunzip -c | docker import - $repo:$tag
+}
+
+worked=1
+for attempt in {1..200}; do
+  [[ $worked != 0]] || break
+  docker_import && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "fetch: failed to import diff image"
+log "fetch: successfully imported image"

data/snippets/file_docker_import.erb

@@ -1 +1,15 @@
-cat <%= data[:path] %> | gunzip -c | <%= docker_import(data[:repo], data[:tag]) %>
+docker_import() {
+  path=<%= data[:path] %>
+  repo=<%= data[:repo] %>
+  tag=<%= data[:tag] %>
+  cat $path | gunzip -c | docker import - $repo:$tag
+}
+
+# Hopefully this isn't ever needed
+worked=1
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  docker_import && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "fetch: failed to import image"
+log "fetch: successfully imported image"

data/snippets/get_and_install_deb.erb

@@ -1,2 +1,13 @@
-<%= get_from_s3(data[:s3_url], data[:deb_path]) %>
-<%= install_package(data[:deb_path]) %>
+worked=1
+s3_path="<%= data[:s3_url] %>"
+output_path="<%= data[:deb_path] %>"
+log "fetch: starting to fetch $s3_path"
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  log "fetch: attempt ${attempt} to get $s3_path ..."
+  s3cmd -f get $s3_path $output_path 2> >(log) && worked=0 || (log "fetch: attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "fetch: failed to pull deb from S3"
+log "fetch: successfully fetched $s3_path"
+
+dpkg -i "<%= data[:deb_path] %>"

data/snippets/registry_import.erb

@@ -1 +1,13 @@
-docker pull <%= data[:repo] %>:<%= data[:tag] %>
+docker_pull() {
+  repo=<%= data[:repo] %>
+  tag=<%= data[:tag] %>
+  docker pull $repo:$tag
+}
+
+worked=1
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  docker_pull && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "failed to import image"
+log "fetch: successfully imported image"

data/snippets/s3_diff_docker_import.erb

@@ -3,24 +3,48 @@ base_image = "/opt/dockly/base_image.tar"
 %>
 
 s3_diff_docker_import_base_fn() {
-<%= get_from_s3(data[:base_image]) %>
+  s3_path="<%= data[:base_image] %>"
+  log "fetch: starting to fetch $s3_path"
+  s3cmd -f get $s3_path - 2> >(log)
+  log "fetch: successfully fetched $s3_path"
 }
 
 s3_diff_docker_import_diff_fn() {
-<%= get_from_s3(data[:diff_image]) %>
+  s3_path="<%= data[:diff_image] %>"
+  log "fetch: starting to fetch $s3_path"
+  s3cmd -f get $s3_path - 2> >(log)
+  log "fetch: successfully fetched $s3_path"
+}
+
+base_image() {
+  s3_diff_docker_import_base_fn | gunzip -vc > "<%= base_image %>" 2> >(log)
 }
 
 stream_image() {
-size=$(stat --format "%s" "<%= base_image %>")
-head_size=$(($size - 1024))
-head -c $head_size "<%= base_image %>"
-s3_diff_docker_import_diff_fn | (gunzip -vc 2> >(log) || fatal "tardiff failed to gunzip")
+  size=$(stat --format "%s" "<%= base_image %>")
+  head_size=$(($size - 1024))
+  head -c $head_size "<%= base_image %>"
+  s3_diff_docker_import_diff_fn | (gunzip -vc 2> >(log) || fatal "tardiff failed to gunzip")
 }
 
 docker_import() {
-<%= docker_import(data[:repo], data[:tag]).to_s.strip %> > >(log) 2>&1 || fatal "docker failed to import"
+  repo=<%= data[:repo] %>
+  tag=<%= data[:tag] %>
+  docker import - $repo:$tag > >(log) 2>&1 || fatal "docker failed to import"
 }
 
-s3_diff_docker_import_base_fn | gunzip -vc > "<%= base_image %>" 2> >(log)
+worked=1
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  base_image && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "fetch: failed to pull base image"
+log "fetch: successfully pulled base image"
 
-stream_image | docker_import
+worked=1
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  stream_image | docker_import && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "fetch: failed to import diff image"
+log "fetch: successfully imported diff image"

data/snippets/s3_docker_import.erb

@@ -1,4 +1,20 @@
 s3_docker_import_fn() {
-<%= get_from_s3(data[:s3_url]) %>
+  s3_path="<%= data[:s3_url] %>"
+  log "fetch: starting to fetch $s3_path"
+  s3cmd -f get $s3_path - 2> >(log)
+  log "fetch: successfully fetched $s3_path"
 }
-s3_docker_import_fn | gunzip -c | <%= docker_import(data[:repo], data[:tag]) %>
+
+docker_import() {
+  repo=<%= data[:repo] %>
+  tag=<%= data[:tag] %>
+  s3_docker_import_fn | gunzip -c | docker import - $repo:$tag
+}
+
+worked=1
+for attempt in {1..200}; do
+  [[ $worked != 0 ]] || break
+  docker_import && worked=0 || (log "fetch: attempt $attempt failed, sleeping 30"; sleep 30)
+done
+[[ $worked != 0 ]] && fatal "failed to import image"
+log "fetch: successfully imported image"

data/spec/dockly/bash_builder_spec.rb

@@ -10,67 +10,15 @@ describe Dockly::BashBuilder do
     end
   end
 
-  describe "#get_from_s3" do
-    let(:s3_url) { "s3://url-for-s3/file.tar.gz" }
-    context "uses the default output" do
-      it "polls from s3 and sets the s3_path" do
-        output = subject.get_from_s3(s3_url)
-        expect(output).to include("s3cmd -f get $s3_path $output_path")
-        expect(output).to include("s3_path=\"#{s3_url}\"")
-        expect(output).to include("output_path=\"-\"")
-      end
-    end
-    context "uses a specific output directory" do
-      let(:output_dir) { "test" }
-      it "polls from s3 and sets the s3_path and output_path" do
-        output = subject.get_from_s3(s3_url, output_dir)
-        expect(output).to include("s3cmd -f get $s3_path $output_path")
-        expect(output).to include("s3_path=\"#{s3_url}\"")
-        expect(output).to include("output_path=\"#{output_dir}\"")
-      end
-    end
-  end
-
-  describe "#install_package" do
-    let(:path) { "/opt/dockly/deb.deb" }
-    it "installs from the given path" do
-      output = subject.install_package(path)
-      expect(output.strip).to eq("dpkg -i \"#{path}\"")
-    end
-  end
-
   describe "#get_and_install_deb" do
     let(:s3_url) { "s3://bucket/path-to-deb.deb" }
     let(:deb_path) { "/opt/dockly/deb_path.deb" }
     it "gets from s3 and installs the package" do
-      expect(subject).to receive(:get_from_s3).with(s3_url, deb_path)
-      expect(subject).to receive(:install_package).with(deb_path)
-      subject.get_and_install_deb(s3_url, deb_path)
-    end
-  end
-
-  describe "#docker_import" do
-    context "when not given a repo" do
-      it "imports with no tagging" do
-        output = subject.docker_import
-        expect(output).to include("docker import -")
-      end
-    end
-
-    context "when given a repo" do
-      let(:repo) { "aRepo" }
-      it "imports with repo and latest" do
-        output = subject.docker_import(repo)
-        expect(output).to include("docker import - #{repo}:latest")
-      end
-
-      context "and a non-default tag" do
-        let(:tag) { "aTag" }
-        it "imports with repo and the tag" do
-          output = subject.docker_import(repo, tag)
-          expect(output).to include("docker import - #{repo}:#{tag}")
-        end
-      end
+      output = subject.get_and_install_deb(s3_url, deb_path)
+      expect(output).to include(s3_url)
+      expect(output).to include(deb_path)
+      expect(output).to include("s3cmd -f get")
+      expect(output).to include("dpkg -i")
     end
   end
 
@@ -93,10 +41,10 @@ describe Dockly::BashBuilder do
   describe "#file_docker_import" do
     let(:path) { "/opt/dockly/file.tar.gz" }
     it "cats, gunzips and passes to docker import" do
-      expect(subject).to receive(:docker_import)
       output = subject.file_docker_import(path)
       expect(output).to include("cat")
       expect(output).to include("gunzip -c")
+      expect(output).to include("docker import -")
     end
   end
 
@@ -104,19 +52,23 @@ describe Dockly::BashBuilder do
     let(:base_image) { "s3://bucket/base_image.tar.gz" }
     let(:diff_image) { "/opt/dockly/diff_image.tar.gz" }
     it "gets the base file from S3 and cats that with the diff image and imports to docker" do
-      expect(subject).to receive(:get_from_s3).with(base_image)
-      expect(subject).to receive(:docker_import)
-      expect(subject.file_diff_docker_import(base_image, diff_image)).to include("cat \"#{diff_image}\"")
+      output = subject.file_diff_docker_import(base_image, diff_image)
+      expect(output).to include(base_image)
+      expect(output).to include(diff_image)
+      expect(output).to include("cat \"#{diff_image}\"")
+      expect(output).to include("s3cmd -f get")
+      expect(output).to include("docker import -")
     end
   end
 
   describe "#s3_docker_import" do
     let(:s3_url) { "s3://bucket/image.tar.gz" }
     it "pulls, gunzips and passes to docker import" do
-      expect(subject).to receive(:get_from_s3)
-      expect(subject).to receive(:docker_import)
       output = subject.s3_docker_import(s3_url)
+      expect(output).to include(s3_url)
       expect(output).to include("gunzip -c")
+      expect(output).to include("s3cmd -f get")
+      expect(output).to include("docker import -")
     end
   end
 
@@ -124,13 +76,15 @@ describe Dockly::BashBuilder do
     let(:base_image) { "s3://bucket/base_image.tar.gz" }
     let(:diff_image) { "s3://bucket/diff_image.tar.gz" }
     it "makes two functions for getting from s3, finds the size, and imports both to docker" do
-      expect(subject).to receive(:get_from_s3).twice
-      expect(subject).to receive(:docker_import)
       output = subject.s3_diff_docker_import(base_image, diff_image)
+      expect(output).to include(base_image)
+      expect(output).to include(diff_image)
       expect(output).to include("stat --format \"%s\"")    # get file size
       expect(output).to include("$(($size - 1024))") # compute file size
       expect(output).to include("head -c $head_size")
       expect(output).to include("gunzip")
+      expect(output).to include("s3cmd -f get")
+      expect(output).to include("docker import -")
     end
   end
 
@@ -139,7 +93,9 @@ describe Dockly::BashBuilder do
     context "not given a tag" do
       it "pulls the latest" do
         output = subject.registry_import(repo)
-        expect(output).to include("docker pull #{repo}:latest")
+        expect(output).to include("docker pull $repo:$tag")
+        expect(output).to include("repo=#{repo}")
+        expect(output).to include("tag=latest")
       end
     end
 
@@ -147,7 +103,9 @@ describe Dockly::BashBuilder do
       let(:tag) { "aTag" }
       it "pulls to specified tag" do
         output = subject.registry_import(repo, tag)
-        expect(output).to include("docker pull #{repo}:#{tag}")
+        expect(output).to include("docker pull $repo:$tag")
+        expect(output).to include("repo=#{repo}")
+        expect(output).to include("tag=#{tag}")
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dockly
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-16 00:00:00.000000000 Z
+date: 2015-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clamp
@@ -278,7 +278,6 @@ files:
 - lib/dockly/version.rb
 - lib/foreman/cli_fix.rb
 - lib/foreman/export/base_fix.rb
-- snippets/docker_import.erb
 - snippets/docker_tag_latest.erb
 - snippets/file_diff_docker_import.erb
 - snippets/file_docker_import.erb

data/snippets/docker_import.erb

@@ -1 +0,0 @@
-docker import - <% if data[:repo] %><%= data[:repo] %>:<%= data[:tag] %><% end %>