contained_mr 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a671efb4fb8421ddf841a9eeabb64b5612bf930
-  data.tar.gz: 304aff98010a52db78225a845406f0cb19bd0651
+  metadata.gz: 440ccef564b310bd2bc7752380b7d97861ab2020
+  data.tar.gz: 15f09bb85ba6e21447b38b26d0790b09342d3f01
 SHA512:
-  metadata.gz: d558c08cce053336b116dda4a270c8d391de24d1b1d7ddecaf4c7761b0cb5ac227dbc7952f268c9b6a1448fd935290510d6e34ff061e09d203b2758fb0b14f0f
-  data.tar.gz: 388ef7ed32cf7506b732cabe66b8f38b869014f89d2b6916e6fc955133c953d6811cbce86daba9fc037746af68063c3fa96738ff6c8dd5eb5fcd9993fd37e8fd
+  metadata.gz: 814c12447b2d338a6781cc81419117b189c19598f9dc942f0f8264dd05dfbd4d4cab297ee029a6efe9808cb5b5366cdb0968c6617ba9165eda169c76f2cc0b24
+  data.tar.gz: e4b768cb0013ea3dc9b8f9817481eda342c87d966cb4dfcc206f175a5112826960d1c540e53ceaf1729f401e67c3ffb3dab38857e66ff7acda747f36c1f34c61

data/README.md

@@ -33,6 +33,17 @@ be executed in every shell where `contained-mr` is used.
 eval "$(docker-machine env dev)"
 ```
 
+### Cleanup
+
+When tests go wrong, they often leave dead containers and images behind.
+Removing all stopped containers and all unused images is a big stick that works
+quite well in development environments.
+
+```bash
+docker ps --all --quiet --no-trunc | xargs docker rm
+docker images --quiet --no-trunc | xargs docker rmi
+```
+
 
 ## Contributing to contained_mr
 

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.3.2

data/contained_mr.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: contained_mr 0.3.1 ruby lib
+# stub: contained_mr 0.3.2 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "contained_mr"
-  s.version = "0.3.1"
+  s.version = "0.3.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Victor Costan"]
-  s.date = "2015-10-09"
+  s.date = "2015-10-12"
   s.description = "Plumbing for running mappers and reducers inside Docker containers"
   s.email = "victor@costan.us"
   s.extra_rdoc_files = [
@@ -63,6 +63,7 @@ Gem::Specification.new do |s|
     "testdata/hello/mapreduced.yml",
     "testdata/hello/reducer.sh",
     "testdata/input.hello",
+    "testdata/invalid/Dockerfile",
     "testdata/job.hello"
   ]
   s.homepage = "http://github.com/pwnall/contained_mr"

data/lib/contained_mr/job.rb

@@ -65,7 +65,8 @@ class ContainedMr::Job
     end
 
     tar_io = mapper_tar_context mapper_input
-    image = Docker::Image.build_from_tar tar_io, t: mapper_image_tag
+    image = Docker::Image.build_from_tar tar_io, t: mapper_image_tag,
+                                                 forcerm: 1
     @mapper_image_id = image.id
   end
 
@@ -81,7 +82,8 @@ class ContainedMr::Job
     end
 
     tar_io = reducer_tar_context
-    image = Docker::Image.build_from_tar tar_io, t: reducer_image_tag
+    image = Docker::Image.build_from_tar tar_io, t: reducer_image_tag,
+                                                 forcerm: 1
     @reducer_image_id = image.id
   end
 

data/lib/contained_mr/template.rb

@@ -77,7 +77,7 @@ class ContainedMr::Template
   #
   # @param {IO} tar_io IO implementation that produces the image's .tar file
   def build_image(tar_io)
-    image = Docker::Image.build_from_tar tar_io, t: image_tag
+    image = Docker::Image.build_from_tar tar_io, t: image_tag, forcerm: 1
     @image_id = image.id
   end
   private :build_image

data/test/helper.rb

@@ -33,15 +33,17 @@ require 'contained_mr'
 class MiniTest::Test
 end
 
-File.unlink 'testdata/hello.zip' if File.file?('testdata/hello.zip')
-Zip::File.open('testdata/hello.zip', Zip::File::CREATE) do |zip|
-  files = Dir.chdir('testdata/hello') { Dir.glob('**/*') }.sort
-  files.each do |file|
-    path = File.join 'testdata/hello', file
-    if File.directory? path
-      zip.mkdir file
-    elsif File.file? path
-      zip.add file, path
+['hello', 'invalid'].each do |dir|
+  File.unlink "testdata/#{dir}.zip" if File.file?("testdata/#{dir}.zip")
+  Zip::File.open("testdata/#{dir}.zip", Zip::File::CREATE) do |zip|
+    files = Dir.chdir("testdata/#{dir}") { Dir.glob('**/*') }.sort
+    files.each do |file|
+      path = File.join "testdata/#{dir}", file
+      if File.directory? path
+        zip.mkdir file
+      elsif File.file? path
+        zip.add file, path
+      end
     end
   end
 end

data/test/test_job.rb

@@ -31,6 +31,20 @@ class TestJob < MiniTest::Test
     assert_nil @job.reducer_runner, "Reducer started prematurely"
   end
 
+  def test_build_mapper_image_does_not_leak_containers
+    old_containers = Docker::Container.all(all: true)
+    @template.stubs(:mapper_dockerfile).returns(
+       "FROM contained_mrtests/base.hello\nRUN /bin/false\n")
+
+    assert_raises Docker::Error::UnexpectedResponseError do
+      @job.build_mapper_image File.read('testdata/input.hello')
+    end
+
+    assert_equal nil, @job.mapper_image_id
+    containers = Docker::Container.all(all: true)
+    assert_equal old_containers.length, containers.length
+  end
+
   def test_created_mapper_image_tags
     @job.build_mapper_image File.read('testdata/input.hello')
 
@@ -95,6 +109,23 @@ class TestJob < MiniTest::Test
         'contained_mrtests/reducer.testjob:latest'
   end
 
+  def test_build_reducer_image_does_not_leak_containers
+    @job.build_mapper_image File.read('testdata/input.hello')
+    1.upto(3) { |i| @job.run_mapper i }
+
+    old_containers = Docker::Container.all(all: true)
+    @template.stubs(:reducer_dockerfile).returns(
+       "FROM contained_mrtests/base.hello\nRUN /bin/false\n")
+
+    assert_raises Docker::Error::UnexpectedResponseError do
+      @job.build_reducer_image
+    end
+
+    assert_equal nil, @job.reducer_image_id
+    containers = Docker::Container.all(all: true)
+    assert_equal old_containers.length, containers.length
+  end
+
   def test_run_reducer
     @job.build_mapper_image File.read('testdata/input.hello')
     1.upto(3) { |i| @job.run_mapper i }

data/test/test_template.rb

@@ -35,6 +35,19 @@ class TestTemplate < MiniTest::Test
     end
   end
 
+  def test_invalid_template_does_not_leave_containers_behind
+    old_containers = Docker::Container.all(all: true)
+
+    assert_raises Docker::Error::UnexpectedResponseError do
+      template = ContainedMr.new_template 'contained_mrtests', 'invalid',
+          StringIO.new(File.binread('testdata/invalid.zip'))
+      template.destroy
+    end
+
+    containers = Docker::Container.all(all: true)
+    assert_equal old_containers.length, containers.length
+  end
+
   def test_destory_with_two_templates
     template2 = ContainedMr.new_template 'contained_mrtests', 'hello2',
         StringIO.new(File.binread('testdata/hello.zip'))

data/testdata/invalid/Dockerfile

@@ -0,0 +1,5 @@
+FROM busybox
+
+RUN mkdir -p /usr/mrd && chmod 777 /usr/mrd
+COPY . /usr/mrd/
+RUN /bin/false

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contained_mr
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Victor Costan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-09 00:00:00.000000000 Z
+date: 2015-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api
@@ -188,6 +188,7 @@ files:
 - testdata/hello/mapreduced.yml
 - testdata/hello/reducer.sh
 - testdata/input.hello
+- testdata/invalid/Dockerfile
 - testdata/job.hello
 homepage: http://github.com/pwnall/contained_mr
 licenses: