contained_mr 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a2802a380e19e314a40493959bace7208a0a8109
-  data.tar.gz: fb912e50f9e146624364edda1272faeff971f133
+  metadata.gz: 0939e46065ffcf543130f1ed6243a896482a286c
+  data.tar.gz: 3dee0167b3909de88614c152777feb90673e3874
 SHA512:
-  metadata.gz: 82d975a57ef901d4909d8dce6f92d240725fb3ad09d8c3e8f0614c51d98df0cdfbc6c7435bffeb1f734b40025d9487bb89f7e2c40c2e41fb9433d48df2b6238a
-  data.tar.gz: 6f034abfe4bf32dd2ab17ac17c89eab1c7b6d3f1cf2e4bd4296288c638421ec89f3d97cf8252836ef88d1473fe8e4bf166399f1cbf6f51a56816d2597db19981
+  metadata.gz: 9d1296d5f4926d089a5669028f71cba8486575463e0eb2b0de3a264c6e05d8f27a014270b29e30990307681063a39dedfccef6e596d6ba83c577897e56a11269
+  data.tar.gz: 1d2a656d0b2001fa683ee76e4b9e59294ea1a315ef8cea99770fffd2daafc43e3634dc34b42ab28a75b3bc68b3d0c4e97faa508e83250a829949c39d14716e2a

data/.travis.yml

@@ -3,5 +3,11 @@ language: ruby
 services:
   - docker
 rvm:
-  - 2.2.2
-script: rake test
+  - 2.2.4
+  - 2.3.0
+before_install:
+  - gem install bundler
+before_script:
+  - bundle install --jobs=3 --retry=3 --deployment
+script:
+  - rake test

data/Gemfile

@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
-gem 'docker-api', '>= 1.22.4', require: 'docker'
+gem 'docker-api', '>= 1.24.1', require: 'docker'
 gem 'rubyzip', '>= 1.1.7', require: 'zip'
 
 # Add dependencies to develop your gem here.
@@ -8,7 +8,7 @@ gem 'rubyzip', '>= 1.1.7', require: 'zip'
 group :development do
   gem 'bundler', '>= 1.6.1'
   gem 'jeweler', '>= 2.0.1'
-  gem 'minitest', '>= 5.8.0'
+  gem 'minitest', '>= 5.8.3'
   gem 'mocha', '>= 1.1.0'
   gem 'rdoc', '>= 4.2.0'
   gem 'simplecov', '>= 0.10.0'

data/Gemfile.lock

@@ -1,28 +1,27 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.3.8)
+    addressable (2.4.0)
     builder (3.2.2)
     descendants_tracker (0.0.4)
       thread_safe (~> 0.3, >= 0.3.1)
     docile (1.1.5)
-    docker-api (1.22.4)
+    docker-api (1.24.1)
       excon (>= 0.38.0)
       json
     excon (0.45.4)
-    faraday (0.9.1)
+    faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
     git (1.2.9.1)
-    github_api (0.12.4)
-      addressable (~> 2.3)
+    github_api (0.13.1)
+      addressable (~> 2.4.0)
       descendants_tracker (~> 0.0.4)
       faraday (~> 0.8, < 0.10)
       hashie (>= 3.4)
       multi_json (>= 1.7.5, < 2.0)
-      nokogiri (~> 1.6.6)
       oauth2
-    hashie (3.4.2)
-    highline (1.7.5)
+    hashie (3.4.3)
+    highline (1.7.8)
     jeweler (2.0.1)
       builder
       bundler (>= 1.0)
@@ -33,17 +32,17 @@ GEM
       rake
       rdoc
     json (1.8.3)
-    jwt (1.5.1)
+    jwt (1.5.2)
     metaclass (0.0.4)
-    mini_portile (0.6.2)
-    minitest (5.8.0)
+    mini_portile2 (2.0.0)
+    minitest (5.8.3)
     mocha (1.1.0)
       metaclass (~> 0.0.1)
     multi_json (1.11.2)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
-    nokogiri (1.6.6.2)
-      mini_portile (~> 0.6.0)
+    nokogiri (1.6.7.1)
+      mini_portile2 (~> 2.0.0.rc2)
     oauth2 (1.0.0)
       faraday (>= 0.8, < 0.10)
       jwt (~> 1.0)
@@ -52,9 +51,10 @@ GEM
       rack (~> 1.2)
     rack (1.6.4)
     rake (10.4.2)
-    rdoc (4.2.0)
+    rdoc (4.2.1)
+      json (~> 1.4)
     rubyzip (1.1.7)
-    simplecov (0.10.0)
+    simplecov (0.11.1)
       docile (~> 1.1.0)
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
@@ -67,9 +67,9 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (>= 1.6.1)
-  docker-api (>= 1.22.4)
+  docker-api (>= 1.24.1)
   jeweler (>= 2.0.1)
-  minitest (>= 5.8.0)
+  minitest (>= 5.8.3)
   mocha (>= 1.1.0)
   rdoc (>= 4.2.0)
   rubyzip (>= 1.1.7)
@@ -77,4 +77,4 @@ DEPENDENCIES
   yard (>= 0.8.7.6)
 
 BUNDLED WITH
-   1.10.6
+   1.11.2

data/README.md

@@ -5,32 +5,41 @@
 
 Map-Reduce where both the mappers and the reducer run inside Docker containers.
 
+
 ## Development Environment
 
-`contained-mr` requires access to a Docker daemon. The easiest way to
+`contained-mr` requires access to a Docker Swarm daemon. The easiest way to
 bring up a development setup is to install
-[Docker Machine](https://github.com/docker/machine) and
+[Docker Machine](https://github.com/docker/machine),
+[Docker Swarm](https://github.com/docker/swarm) and
 [VirtualBox](https://www.virtualbox.org/).
 
 The commands below install the prerequisites on OSX using
 [Homebrew](http://brew.sh/).
 
 ```bash
-brew install brew-cask docker docker-machine
+brew install docker docker-machine docker-swarm
+brew tap Caskroom/cask
 brew cask install virtualbox
 ```
 
-Create a Docker VM. This is a one-time setup.
+Create a Docker Swarm cluster. This is a one-time setup.
 
 ```bash
-docker-machine create --driver virtualbox --engine-storage-driver overlay dev
+TOKEN=$(docker-swarm create)
+docker-machine create --driver virtualbox --engine-storage-driver overlay \
+    --swarm --swarm-master --swarm-discovery "token://$TOKEN" swarm-master
+docker-machine create --driver virtualbox --engine-storage-driver overlay \
+    --swarm --swarm-discovery "token://$TOKEN" swarm-agent-1
+docker-machine create --driver virtualbox --engine-storage-driver overlay \
+    --swarm --swarm-discovery "token://$TOKEN" swarm-agent-2
 ```
 
-Set up the local environment to point to the Docker daemon in the VM. This must
-be executed in every shell where `contained-mr` is used.
+Set up the local environment to point to the Swarm master. This must be
+executed in every shell where `contained-mr` is used.
 
 ```bash
-eval "$(docker-machine env dev)"
+eval "$(docker-machine env --swarm swarm-master)"
 ```
 
 ### Cleanup
@@ -44,6 +53,29 @@ docker ps --all --quiet --no-trunc | xargs docker rm
 docker images --quiet --no-trunc | xargs docker rmi
 ```
 
+### Testing Against Experimental Docker
+
+This cluster setup will use the development version of Docker.
+
+```bash
+TOKEN=$(docker-swarm create)
+ISO_URL=$(curl \
+    https://api.github.com/repos/ahbeng/boot2docker-experimental/releases/latest \
+    | grep -o https://.*/boot2docker.iso)
+docker-machine create --driver virtualbox --engine-storage-driver overlay \
+    --swarm --swarm-master --swarm-discovery "token://$TOKEN" \
+    --swarm-image "dockerswarm/swarm:master" \
+    --virtualbox-boot2docker-url $ISO_URL swarm-master
+docker-machine create --driver virtualbox --engine-storage-driver overlay \
+    --swarm --swarm-discovery "token://$TOKEN" \
+    --swarm-image "dockerswarm/swarm:master" \
+    --virtualbox-boot2docker-url $ISO_URL swarm-agent-1
+docker-machine create --driver virtualbox --engine-storage-driver overlay \
+    --swarm --swarm-discovery "token://$TOKEN" \
+    --swarm-image "dockerswarm/swarm:master" \
+    --virtualbox-boot2docker-url $ISO_URL swarm-agent-2
+```
+
 
 ## Contributing to contained_mr
 

data/VERSION

@@ -1 +1 @@
-0.4.1
+0.5.0

data/contained_mr.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: contained_mr 0.4.1 ruby lib
+# stub: contained_mr 0.5.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "contained_mr"
-  s.version = "0.4.1"
+  s.version = "0.5.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Victor Costan"]
-  s.date = "2015-10-27"
+  s.date = "2016-01-29"
   s.description = "Plumbing for running mappers and reducers inside Docker containers"
   s.email = "victor@costan.us"
   s.extra_rdoc_files = [
@@ -68,39 +68,39 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/pwnall/contained_mr"
   s.licenses = ["MIT"]
-  s.rubygems_version = "2.4.5.1"
+  s.rubygems_version = "2.5.1"
   s.summary = "Map-Reduce with Docker containers"
 
   if s.respond_to? :specification_version then
     s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<docker-api>, [">= 1.22.4"])
+      s.add_runtime_dependency(%q<docker-api>, [">= 1.24.1"])
       s.add_runtime_dependency(%q<rubyzip>, [">= 1.1.7"])
       s.add_development_dependency(%q<bundler>, [">= 1.6.1"])
       s.add_development_dependency(%q<jeweler>, [">= 2.0.1"])
-      s.add_development_dependency(%q<minitest>, [">= 5.8.0"])
+      s.add_development_dependency(%q<minitest>, [">= 5.8.3"])
       s.add_development_dependency(%q<mocha>, [">= 1.1.0"])
       s.add_development_dependency(%q<rdoc>, [">= 4.2.0"])
       s.add_development_dependency(%q<simplecov>, [">= 0.10.0"])
       s.add_development_dependency(%q<yard>, [">= 0.8.7.6"])
     else
-      s.add_dependency(%q<docker-api>, [">= 1.22.4"])
+      s.add_dependency(%q<docker-api>, [">= 1.24.1"])
       s.add_dependency(%q<rubyzip>, [">= 1.1.7"])
       s.add_dependency(%q<bundler>, [">= 1.6.1"])
       s.add_dependency(%q<jeweler>, [">= 2.0.1"])
-      s.add_dependency(%q<minitest>, [">= 5.8.0"])
+      s.add_dependency(%q<minitest>, [">= 5.8.3"])
       s.add_dependency(%q<mocha>, [">= 1.1.0"])
       s.add_dependency(%q<rdoc>, [">= 4.2.0"])
       s.add_dependency(%q<simplecov>, [">= 0.10.0"])
       s.add_dependency(%q<yard>, [">= 0.8.7.6"])
     end
   else
-    s.add_dependency(%q<docker-api>, [">= 1.22.4"])
+    s.add_dependency(%q<docker-api>, [">= 1.24.1"])
     s.add_dependency(%q<rubyzip>, [">= 1.1.7"])
     s.add_dependency(%q<bundler>, [">= 1.6.1"])
     s.add_dependency(%q<jeweler>, [">= 2.0.1"])
-    s.add_dependency(%q<minitest>, [">= 5.8.0"])
+    s.add_dependency(%q<minitest>, [">= 5.8.3"])
     s.add_dependency(%q<mocha>, [">= 1.1.0"])
     s.add_dependency(%q<rdoc>, [">= 4.2.0"])
     s.add_dependency(%q<simplecov>, [">= 0.10.0"])

data/lib/contained_mr/job_logic.rb

@@ -108,6 +108,8 @@ module ContainedMr::JobLogic
       'MemorySwappiness' => 0,
       'CpuPeriod' => cpu_period,
       'CpuQuota' => (job_section[:vcpus] * cpu_period).to_i,
+      # NOTE: This interpretation of CpuShares only works on Docker Swarm.
+      'CpuShares' => job_section[:vcpus],
       'LogConfig' => {
         'Type' => 'json-file',
         'Config' => {

data/test/test_job_logic.rb

@@ -41,8 +41,9 @@ class TestJobLogic < MiniTest::Test
         'Memory' => 256.5 * 1024 * 1024,
         'MemorySwap' => (256.5 + 64) * 1024 * 1024,
         'MemorySwappiness' => 0,
-        'CpuQuota' => 1500000,
+        'CpuQuota' => 1000000,
         'CpuPeriod' => 1000000,
+        'CpuShares' => 1,
         'LogConfig' => {
           'Type' => 'json-file',
           'Config' => {
@@ -72,8 +73,9 @@ class TestJobLogic < MiniTest::Test
         'Memory' => 768.5 * 1024 * 1024,
         'MemorySwap' => 768.5 * 1024 * 1024,
         'MemorySwappiness' => 0,
-        'CpuQuota' => 500000,
+        'CpuQuota' => 1000000,
         'CpuPeriod' => 1000000,
+        'CpuShares' => 1,
         'LogConfig' => {
           'Type' => 'json-file',
           'Config' => {

data/test/test_mock_runner.rb

@@ -18,8 +18,9 @@ class TestMockRunner < MiniTest::Test
         'Memory' => 256.5 * 1024 * 1024,
         'MemorySwap' => (256.5 + 64) * 1024 * 1024,
         'MemorySwappiness' => 0,
-        'CpuQuota' => 1500000,
+        'CpuQuota' => 1000000,
         'CpuPeriod' => 1000000,
+        'CpuShares' => 1,
         'LogConfig' => {
           'Type' => 'json-file',
           'Config' => {
@@ -114,7 +115,7 @@ class TestMockRunner < MiniTest::Test
   def test_resources
     assert_equal 256.5, @runner._ram_limit
     assert_equal 64, @runner._swap_limit
-    assert_equal 1.5, @runner._vcpus
+    assert_equal 1, @runner._vcpus
     assert_equal 4.5, @runner._logs
   end
 
@@ -134,7 +135,7 @@ class TestMockRunner < MiniTest::Test
                                            '/usr/mrd/map-output'
     assert_equal nil, runner._ram_limit
     assert_equal nil, runner._swap_limit
-    assert_equal 1.5, runner._vcpus
+    assert_equal 1, runner._vcpus
     assert_equal 4.5, runner._logs
   end
 
@@ -144,7 +145,7 @@ class TestMockRunner < MiniTest::Test
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal nil, runner._swap_limit
-    assert_equal 1.5, runner._vcpus
+    assert_equal 1, runner._vcpus
     assert_equal 4.5, runner._logs
   end
 
@@ -174,7 +175,7 @@ class TestMockRunner < MiniTest::Test
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 64, runner._swap_limit
-    assert_equal 1.5, runner._vcpus
+    assert_equal 1, runner._vcpus
     assert_equal nil, runner._logs
   end
 
@@ -184,7 +185,7 @@ class TestMockRunner < MiniTest::Test
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 64, runner._swap_limit
-    assert_equal 1.5, runner._vcpus
+    assert_equal 1, runner._vcpus
     assert_equal nil, runner._logs
   end
 
@@ -194,7 +195,7 @@ class TestMockRunner < MiniTest::Test
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 64, runner._swap_limit
-    assert_equal 1.5, runner._vcpus
+    assert_equal 1, runner._vcpus
     assert_equal nil, runner._logs
   end
 
@@ -204,7 +205,7 @@ class TestMockRunner < MiniTest::Test
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 0, runner._swap_limit
-    assert_equal 1.5, runner._vcpus
+    assert_equal 1, runner._vcpus
     assert_equal 4.5, runner._logs
   end
 end

data/test/test_runner_logic.rb

@@ -17,8 +17,9 @@ class TestRunnerLogic < MiniTest::Test
         'Memory' => 256.5 * 1024 * 1024,
         'MemorySwap' => (256.5 + 64) * 1024 * 1024,
         'MemorySwappiness' => 0,
-        'CpuQuota' => 1500000,
+        'CpuQuota' => 1000000,
         'CpuPeriod' => 1000000,
+        'CpuShares' => 1,
         'LogConfig' => {
           'Type' => 'json-file',
           'Config' => {

data/testdata/job.hello

@@ -1,7 +1,7 @@
 {
   "mapper": {
     "wait_time": 2.5,
-    "vcpus": 1.5,
+    "vcpus": 1,
     "ram": 256.5,
     "swap": 64,
     "logs": 4.5,
@@ -9,7 +9,7 @@
   },
   "reducer": {
     "wait_time": 2,
-    "vcpus": 0.5,
+    "vcpus": 1,
     "ram": 768.5,
     "swap": 0,
     "logs": 6.5,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contained_mr
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Victor Costan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-27 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.22.4
+        version: 1.24.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.22.4
+        version: 1.24.1
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.8.0
+        version: 5.8.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.8.0
+        version: 5.8.3
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -210,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Map-Reduce with Docker containers