contained_mr 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0939e46065ffcf543130f1ed6243a896482a286c
-  data.tar.gz: 3dee0167b3909de88614c152777feb90673e3874
+  metadata.gz: b85709886ebd903cf0ce3ebc0843fd1b9122faff
+  data.tar.gz: d9a4f3fb295ea57d39b7ee5f21526c5ed7903d79
 SHA512:
-  metadata.gz: 9d1296d5f4926d089a5669028f71cba8486575463e0eb2b0de3a264c6e05d8f27a014270b29e30990307681063a39dedfccef6e596d6ba83c577897e56a11269
-  data.tar.gz: 1d2a656d0b2001fa683ee76e4b9e59294ea1a315ef8cea99770fffd2daafc43e3634dc34b42ab28a75b3bc68b3d0c4e97faa508e83250a829949c39d14716e2a
+  metadata.gz: dd44006c2a201c54d0aeee2fc3a4815c231acfddebd87cdb8b022b3be3bd0152ff0a3dfa95197b71dd4e4f5e048881ec6864e40ac7274809e42cd6d3f2e42fcd
+  data.tar.gz: 8118a13d9c80711bff4ec08277bdef4719a4f89c2e81779a3c5c8d6a12933bc91978167fb2cb06dc0cbe276f864b3a23bc1c24a03490aa060399290e6ec96f2b

data/.travis.yml

@@ -1,13 +1,16 @@
 sudo: required
+dist: trusty
 language: ruby
-services:
-  - docker
 rvm:
   - 2.2.4
   - 2.3.0
 before_install:
+  - sudo apt-get update
+  - sudo apt-get install -o Dpkg::Options::="--force-confold" --force-yes -y docker-engine
   - gem install bundler
 before_script:
   - bundle install --jobs=3 --retry=3 --deployment
+  - docker version
+  - docker info
 script:
-  - rake test
+  - bundle exec rake test

data/Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'docker-api', '>= 1.24.1', require: 'docker'
+gem 'docker-api', '>= 1.26.2', require: 'docker'
 gem 'rubyzip', '>= 1.1.7', require: 'zip'
 
 # Add dependencies to develop your gem here.
@@ -10,6 +10,7 @@ group :development do
   gem 'jeweler', '>= 2.0.1'
   gem 'minitest', '>= 5.8.3'
   gem 'mocha', '>= 1.1.0'
+  gem 'rake', '>= 11.1.2'
   gem 'rdoc', '>= 4.2.0'
   gem 'simplecov', '>= 0.10.0'
   gem 'yard', '>= 0.8.7.6'

data/Gemfile.lock

@@ -6,13 +6,13 @@ GEM
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     docile (1.1.5)
-    docker-api (1.24.1)
+    docker-api (1.26.2)
       excon (>= 0.38.0)
       json
-    excon (0.45.4)
+    excon (0.48.0)
     faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
-    git (1.2.9.1)
+    git (1.3.0)
     github_api (0.13.1)
       addressable (~> 2.4.0)
       descendants_tracker (~> 0.0.4)
@@ -32,29 +32,29 @@ GEM
       rake
       rdoc
     json (1.8.3)
-    jwt (1.5.2)
+    jwt (1.5.1)
     metaclass (0.0.4)
     mini_portile2 (2.0.0)
-    minitest (5.8.3)
+    minitest (5.8.4)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
     multi_json (1.11.2)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
-    nokogiri (1.6.7.1)
+    nokogiri (1.6.7.2)
       mini_portile2 (~> 2.0.0.rc2)
-    oauth2 (1.0.0)
+    oauth2 (1.1.0)
       faraday (>= 0.8, < 0.10)
-      jwt (~> 1.0)
+      jwt (~> 1.0, < 1.5.2)
       multi_json (~> 1.3)
       multi_xml (~> 0.5)
-      rack (~> 1.2)
+      rack (>= 1.2, < 3)
     rack (1.6.4)
-    rake (10.4.2)
-    rdoc (4.2.1)
+    rake (11.1.2)
+    rdoc (4.2.2)
       json (~> 1.4)
-    rubyzip (1.1.7)
-    simplecov (0.11.1)
+    rubyzip (1.2.0)
+    simplecov (0.11.2)
       docile (~> 1.1.0)
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
@@ -67,10 +67,11 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (>= 1.6.1)
-  docker-api (>= 1.24.1)
+  docker-api (>= 1.26.2)
   jeweler (>= 2.0.1)
   minitest (>= 5.8.3)
   mocha (>= 1.1.0)
+  rake (>= 11.1.2)
   rdoc (>= 4.2.0)
   rubyzip (>= 1.1.7)
   simplecov (>= 0.10.0)

data/README.md

@@ -6,6 +6,13 @@
 Map-Reduce where both the mappers and the reducer run inside Docker containers.
 
 
+## Prerequisites
+
+`contained-mr` requires [docker](https://github.com/docker/docker) version 1.9
+or above. If used, [docker-swarm](https://github.com/docker/swarm) must be at
+vesion 1.0 or above.
+
+
 ## Development Environment
 
 `contained-mr` requires access to a Docker Swarm daemon. The easiest way to

data/Rakefile

@@ -29,7 +29,8 @@ require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
+  test.verbose = false
+  test.warning = false
 end
 
 desc "Code coverage detail"

data/VERSION

@@ -1 +1 @@
-0.5.0
+0.6.0

data/contained_mr.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: contained_mr 0.5.0 ruby lib
+# stub: contained_mr 0.6.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "contained_mr"
-  s.version = "0.5.0"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Victor Costan"]
-  s.date = "2016-01-29"
+  s.date = "2016-03-28"
   s.description = "Plumbing for running mappers and reducers inside Docker containers"
   s.email = "victor@costan.us"
   s.extra_rdoc_files = [
@@ -75,33 +75,36 @@ Gem::Specification.new do |s|
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<docker-api>, [">= 1.24.1"])
+      s.add_runtime_dependency(%q<docker-api>, [">= 1.26.2"])
       s.add_runtime_dependency(%q<rubyzip>, [">= 1.1.7"])
       s.add_development_dependency(%q<bundler>, [">= 1.6.1"])
       s.add_development_dependency(%q<jeweler>, [">= 2.0.1"])
       s.add_development_dependency(%q<minitest>, [">= 5.8.3"])
       s.add_development_dependency(%q<mocha>, [">= 1.1.0"])
+      s.add_development_dependency(%q<rake>, [">= 11.1.2"])
       s.add_development_dependency(%q<rdoc>, [">= 4.2.0"])
       s.add_development_dependency(%q<simplecov>, [">= 0.10.0"])
       s.add_development_dependency(%q<yard>, [">= 0.8.7.6"])
     else
-      s.add_dependency(%q<docker-api>, [">= 1.24.1"])
+      s.add_dependency(%q<docker-api>, [">= 1.26.2"])
       s.add_dependency(%q<rubyzip>, [">= 1.1.7"])
       s.add_dependency(%q<bundler>, [">= 1.6.1"])
       s.add_dependency(%q<jeweler>, [">= 2.0.1"])
       s.add_dependency(%q<minitest>, [">= 5.8.3"])
       s.add_dependency(%q<mocha>, [">= 1.1.0"])
+      s.add_dependency(%q<rake>, [">= 11.1.2"])
       s.add_dependency(%q<rdoc>, [">= 4.2.0"])
       s.add_dependency(%q<simplecov>, [">= 0.10.0"])
       s.add_dependency(%q<yard>, [">= 0.8.7.6"])
     end
   else
-    s.add_dependency(%q<docker-api>, [">= 1.24.1"])
+    s.add_dependency(%q<docker-api>, [">= 1.26.2"])
     s.add_dependency(%q<rubyzip>, [">= 1.1.7"])
     s.add_dependency(%q<bundler>, [">= 1.6.1"])
     s.add_dependency(%q<jeweler>, [">= 2.0.1"])
     s.add_dependency(%q<minitest>, [">= 5.8.3"])
     s.add_dependency(%q<mocha>, [">= 1.1.0"])
+    s.add_dependency(%q<rake>, [">= 11.1.2"])
     s.add_dependency(%q<rdoc>, [">= 4.2.0"])
     s.add_dependency(%q<simplecov>, [">= 0.10.0"])
     s.add_dependency(%q<yard>, [">= 0.8.7.6"])

data/lib/contained_mr/job.rb

@@ -65,9 +65,11 @@ class ContainedMr::Job
     end
 
     tar_io = mapper_tar_context mapper_input
-    image = Docker::Image.build_from_tar tar_io, t: mapper_image_tag,
-                                                 forcerm: 1
-    @mapper_image_id = image.id
+    Docker::Image.build_from_tar tar_io, mapper_image_options
+
+    # NOTE: The build process returns a short image ID. We need to perform
+    #       another API call to get the canonical ID.
+    @mapper_image_id = Docker::Image.get(mapper_image_tag).id
   end
 
   # Builds the Docker image used to run this job's reducer.
@@ -82,9 +84,11 @@ class ContainedMr::Job
     end
 
     tar_io = reducer_tar_context
-    image = Docker::Image.build_from_tar tar_io, t: reducer_image_tag,
-                                                 forcerm: 1
-    @reducer_image_id = image.id
+    Docker::Image.build_from_tar tar_io, reducer_image_options
+
+    # NOTE: The build process returns a short image ID. We need to perform
+    #       another API call to get the canonical ID.
+    @reducer_image_id = Docker::Image.get(reducer_image_tag).id
   end
 
   # Runs one of the job's mappers.

data/lib/contained_mr/job_logic.rb

@@ -49,6 +49,24 @@ module ContainedMr::JobLogic
     "#{@name_prefix}/reducer.#{@id}"
   end
 
+  # @return {Hash<Symbol, Object>} options passed to the Docker API when
+  #   building when building the mapper image
+  def mapper_image_options
+    {
+      t: mapper_image_tag, forcerm: 1,
+      buildargs: { 'affinity:image' => "=#{@template.image_tag}" }.to_json
+    }
+  end
+
+  # @return {Hash<Symbol, Object>} options passed to the Docker API when
+  #   building when building the mapper image
+  def reducer_image_options
+    {
+      t: reducer_image_tag, forcerm: 1,
+      buildargs: { 'affinity:image' => "=#{@template.image_tag}" }.to_json
+    }
+  end
+
   # @return {Hash<String, Object>} params used to create a mapper container
   def mapper_container_options(i)
     ulimits = @mapper_options[:ulimits].map do |k, v|

data/lib/contained_mr/template.rb

@@ -77,8 +77,11 @@ class ContainedMr::Template
   #
   # @param {IO} tar_io IO implementation that produces the image's .tar file
   def build_image(tar_io)
-    image = Docker::Image.build_from_tar tar_io, t: image_tag, forcerm: 1
-    @image_id = image.id
+    Docker::Image.build_from_tar tar_io, t: image_tag, forcerm: 1
+
+    # NOTE: The build process returns a short image ID. We need to perform
+    #       another API call to get the canonical ID.
+    @image_id = Docker::Image.get(image_tag).id
   end
   private :build_image
 end

data/lib/contained_mr/template_logic.rb

@@ -70,6 +70,7 @@ module ContainedMr::TemplateLogic
   def job_dockerfile(job_definition, input_source)
     <<DOCKER_END
 FROM #{image_tag}
+ARG affinity:image=
 COPY #{input_source} #{job_definition['input'] || '/input'}
 WORKDIR #{job_definition['chdir'] || '/'}
 ENTRYPOINT #{JSON.dump(job_definition['cmd'] || ['/bin/sh'])}

data/test/test_job.rb

@@ -23,7 +23,7 @@ class TestJob < MiniTest::Test
 
     image = Docker::Image.get @job.mapper_image_tag
     assert image, 'Docker::Image'
-    assert_operator image.id, :start_with?, @job.mapper_image_id
+    assert_equal image.id, @job.mapper_image_id
 
     1.upto 3 do |i|
       assert_nil @job.mapper_runner(i), "Mapper #{i} started prematurely"
@@ -48,8 +48,7 @@ class TestJob < MiniTest::Test
   def test_created_mapper_image_tags
     @job.build_mapper_image File.read('testdata/input.hello')
 
-    images = Docker::Image.all
-    image = images.find { |i| i.id.start_with? @job.mapper_image_id }
+    image = Docker::Image.get @job.mapper_image_id
     assert image, 'Docker::Image in collection returned by Docker::Image.all'
     assert image.info['RepoTags'], "Image missing RepoTags: #{image.inspect}"
     assert_includes image.info['RepoTags'],
@@ -97,12 +96,11 @@ class TestJob < MiniTest::Test
     @job.build_reducer_image
     image = Docker::Image.get @job.reducer_image_tag
     assert image, 'Docker::Image'
-    assert_operator image.id, :start_with?, @job.reducer_image_id
+    assert_equal image.id, @job.reducer_image_id
 
     assert_nil @job.reducer_runner, "Reducer started prematurely"
 
-    images = Docker::Image.all
-    image = images.find { |i| i.id.start_with? @job.reducer_image_id }
+    image = Docker::Image.get @job.reducer_image_id
     assert image, 'Docker::Image in collection returned by Docker::Image.all'
     assert image.info['RepoTags'], "Image missing RepoTags: #{image.inspect}"
     assert_includes image.info['RepoTags'],

data/test/test_job_logic.rb

@@ -17,6 +17,22 @@ class TestJobLogic < MiniTest::Test
     assert_equal 'contained_mrtests/reducer.testjob', @job.reducer_image_tag
   end
 
+  def test_mapper_image_options
+    golden = {
+      t: 'contained_mrtests/mapper.testjob', forcerm: 1,
+      buildargs: '{"affinity:image":"=contained_mrtests/base.hello"}'
+    }
+    assert_equal golden, @job.mapper_image_options
+  end
+
+  def test_reducer_image_options
+    golden = {
+      t: 'contained_mrtests/reducer.testjob', forcerm: 1,
+      buildargs: '{"affinity:image":"=contained_mrtests/base.hello"}'
+    }
+    assert_equal golden, @job.reducer_image_options
+  end
+
   def test_mapper_container_options
     assert_equal @template, @job.template
     assert_equal 'contained_mrtests', @job.name_prefix

data/test/test_template.rb

@@ -20,8 +20,7 @@ class TestTemplate < MiniTest::Test
   end
 
   def test_created_image_tags
-    images = Docker::Image.all
-    image = images.find { |i| i.id.start_with? @template.image_id }
+    image = Docker::Image.get @template.image_id
     assert image, 'Docker::Image in collection returned by Docker::Image.all'
     assert image.info['RepoTags'], "Image missing RepoTags: #{image.inspect}"
     assert_includes image.info['RepoTags'],

data/testdata/Dockerfile.hello.mapper

@@ -1,4 +1,5 @@
 FROM contained_mrtests/base.hello
+ARG affinity:image=
 COPY input /usr/mrd/map-input
 WORKDIR /usr/mrd
 ENTRYPOINT ["/bin/sh","/usr/mrd/mapper.sh"]

data/testdata/Dockerfile.hello.reducer

@@ -1,4 +1,5 @@
 FROM contained_mrtests/base.hello
+ARG affinity:image=
 COPY . /usr/mrd/
 WORKDIR /
 ENTRYPOINT ["/bin/sh","/usr/mrd/reducer.sh"]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contained_mr
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Victor Costan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.24.1
+        version: 1.26.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.24.1
+        version: 1.26.2
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.1.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 11.1.2
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement