contained_mr 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e185d4610f9518f8f95924cd6569f8f62bdd309d
-  data.tar.gz: 9dc30131aa6475f07b3e5042e43877dbab91d0a3
+  metadata.gz: a2802a380e19e314a40493959bace7208a0a8109
+  data.tar.gz: fb912e50f9e146624364edda1272faeff971f133
 SHA512:
-  metadata.gz: 597919494dce5409863b1ecd5d8ea9b75870f6705d945e74822995501184b6a0138fdd730a6c04e2a0bbe1bb7c85bcb3028f5a1c6291ec2bd0284bc01d06dfc3
-  data.tar.gz: c0936fd70b787f76aa222186ee6e915b9d400f0abb036af0c09a038c33a6049e4cf11d4435ab3c0dea6291a5fae6feb13a0b7eb6c647e4fa24e4a006f49add3d
+  metadata.gz: 82d975a57ef901d4909d8dce6f92d240725fb3ad09d8c3e8f0614c51d98df0cdfbc6c7435bffeb1f734b40025d9487bb89f7e2c40c2e41fb9433d48df2b6238a
+  data.tar.gz: 6f034abfe4bf32dd2ab17ac17c89eab1c7b6d3f1cf2e4bd4296288c638421ec89f3d97cf8252836ef88d1473fe8e4bf166399f1cbf6f51a56816d2597db19981

data/README.md

@@ -23,7 +23,7 @@ brew cask install virtualbox
 Create a Docker VM. This is a one-time setup.
 
 ```bash
-docker-machine create --driver virtualbox dev
+docker-machine create --driver virtualbox --engine-storage-driver overlay dev
 ```
 
 Set up the local environment to point to the Docker daemon in the VM. This must

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/contained_mr.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: contained_mr 0.4.0 ruby lib
+# stub: contained_mr 0.4.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "contained_mr"
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Victor Costan"]
-  s.date = "2015-10-17"
+  s.date = "2015-10-27"
   s.description = "Plumbing for running mappers and reducers inside Docker containers"
   s.email = "victor@costan.us"
   s.extra_rdoc_files = [

data/lib/contained_mr/cleaner.rb

@@ -14,23 +14,32 @@ class ContainedMr::Cleaner
   end
 
   # Removes all images and containers matching this cleaner's name prefix.
+  #
+  # @return {Array<Docker::Container, Docker::Image>} the removed objects
   def destroy_all!
-    destroy_all_containers!
-    destroy_all_images!
+    containers = destroy_all_containers!
+    images = destroy_all_images!
+    containers + images
   end
 
+  # @return {Array<Docker::Container>} the removed containers
   def destroy_all_containers!
     containers = Docker::Container.all all: true,
                                        filters: container_filters.to_json
     containers.each do |container|
-      container.delete force: false, volumes: true
+      begin
+        container.delete force: false, volumes: true
+      rescue Docker::Error::NotFoundError
+        # Workaround for https://github.com/docker/docker/issues/14474
+      end
     end
   end
-  private :destroy_all_containers!
 
+  # @return {Array<Docker::Image>} the removed images
   def destroy_all_images!
     tag_prefix = "#{@name_prefix}/"
     images = Docker::Image.all
+    deleted_images = []
     images.each do |image|
       next unless image_tags = image.info['RepoTags']
       image_tags.each do |image_tag|
@@ -38,10 +47,11 @@ class ContainedMr::Cleaner
         # HACK(pwnall): Trick docker-api into issuing a DELETE request by tag.
         tag_image = Docker::Image.new Docker.connection, 'id' => image_tag
         tag_image.delete
+        deleted_images << image
       end
     end
+    deleted_images
   end
-  private :destroy_all_images!
 
   # @return { Hash<Symbol, Array<String>> } filters used to identify Docker
   #   containers started by this controller

data/lib/contained_mr/runner.rb

@@ -50,7 +50,11 @@ class ContainedMr::Runner
   def destroy!(container = nil)
     unless @container_id.nil?
       container ||= Docker::Container.get @container_id
-      container.delete force: true
+      begin
+        container.delete force: true
+      rescue Docker::Error::NotFoundError
+        # Workaround for https://github.com/docker/docker/issues/14474
+      end
       @container_id = nil
     end
     self

data/test/test_cleaner.rb

@@ -7,12 +7,30 @@ class TestCleaner < MiniTest::Test
     @job = @template.new_job 'testjob',
         JSON.load(File.read('testdata/job.hello'))
     @job.build_mapper_image File.read('testdata/input.hello')
+    @runner = ContainedMr::Runner.new @job.mapper_container_options(2), 2.5,
+                                      @template.mapper_output_path
+    class <<@runner
+      def destroy!(*args)
+        self
+      end
+    end
+    @runner.perform
 
     @cleaner = ContainedMr::Cleaner.new 'contained_mrtests'
   end
 
-  def test_destroy_all
+  def teardown
+    Docker::Container.any_instance.unstub :delete
     @cleaner.destroy_all!
+  end
+
+  def test_destroy_all
+    objects = @cleaner.destroy_all!
+    assert_equal 3, objects.length
+    assert_kind_of Docker::Container, objects[0]
+    assert_kind_of Docker::Image, objects[1]
+    assert_kind_of Docker::Image, objects[2]
+
     assert_raises Docker::Error::NotFoundError do
       Docker::Image.get @job.mapper_image_tag
     end
@@ -27,7 +45,24 @@ class TestCleaner < MiniTest::Test
     job2 = template2.new_job 'testjob2',
         JSON.load(File.read('testdata/job.hello'))
     job2.build_mapper_image File.read('testdata/input.hello')
-    @cleaner.destroy_all!
+    runner2 = ContainedMr::Runner.new job2.mapper_container_options(2), 2.5,
+                                      template2.mapper_output_path
+    class <<runner2
+      def destroy!(*args)
+        self
+      end
+    end
+    runner2.perform
+
+    objects = @cleaner.destroy_all!
+    assert_equal 6, objects.length
+    assert_kind_of Docker::Container, objects[0]
+    assert_kind_of Docker::Container, objects[1]
+    assert_kind_of Docker::Image, objects[2]
+    assert_kind_of Docker::Image, objects[3]
+    assert_kind_of Docker::Image, objects[4]
+    assert_kind_of Docker::Image, objects[5]
+
     assert_raises Docker::Error::NotFoundError do
       Docker::Image.get job2.mapper_image_tag
     end
@@ -41,4 +76,26 @@ class TestCleaner < MiniTest::Test
       Docker::Image.get @template.image_tag
     end
   end
+
+  def test_destroy_all_containers_with_duplicates_and_exceptions
+    template2 = ContainedMr.new_template 'contained_mrtests', 'hello2',
+        StringIO.new(File.binread('testdata/hello.zip'))
+    job2 = template2.new_job 'testjob2',
+        JSON.load(File.read('testdata/job.hello'))
+    job2.build_mapper_image File.read('testdata/input.hello')
+    runner2 = ContainedMr::Runner.new job2.mapper_container_options(2), 2.5,
+                                      template2.mapper_output_path
+    class <<runner2
+      def destroy!(*args)
+        self
+      end
+    end
+    runner2.perform
+
+    Docker::Container.any_instance.expects(:delete).twice.
+                      raises Docker::Error::NotFoundError
+    containers = @cleaner.destroy_all_containers!
+    assert_equal 2, containers.length
+    containers.each { |c| assert_kind_of Docker::Container, c }
+  end
 end

data/test/test_runner.rb

@@ -68,4 +68,39 @@ class TestRunner < MiniTest::Test
                                      @template.mapper_output_path
     assert_equal runner, runner.destroy!
   end
+
+  def test_destroy_invalid_container
+    runner = ContainedMr::Runner.new @job.mapper_container_options(2), 2.5,
+                                     @template.mapper_output_path
+
+    # HACK(pwnall): This is a dirty way of getting NotFoundError, which is also
+    #               what the docker gem throws in docker#14474.
+    class <<runner
+      alias_method :old_destroy, :destroy!
+      def destroy!(container)
+        old_container_id = @container_id
+        old_destroy container
+        if container
+          @container_id = old_container_id
+          old_destroy container
+        end
+      end
+    end
+
+    assert_equal runner, runner.perform
+
+    assert_equal nil, runner.container_id, 'container still running'
+    assert_operator runner.ended_at - runner.started_at, :<, 1, 'running time'
+    assert_equal 0, runner.status_code, 'status code'
+    assert_equal false, runner.timed_out, 'timed out'
+    assert_equal "2 3\n", runner.stderr, 'Stderr: $ITEM + $ITEMS'
+    assert_equal "2\nmapper input file\nHello world!\n", runner.stdout,
+                 'Stdout: $ITEM + mapper input file + data file'
+    assert_equal "2\n", runner.output, 'Output: ITEM env variable'
+
+    assert_equal runner.ended_at - runner.started_at,
+                 runner.json_file[:ran_for]
+    assert_equal 0, runner.json_file[:exit_code]
+    assert_equal false, runner.json_file[:timed_out]
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contained_mr
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Victor Costan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-17 00:00:00.000000000 Z
+date: 2015-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api