contained_mr 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7dd43659d3e4a97eb0c431a28fd0e18ef0df524e
-  data.tar.gz: c460e444b84ebd56df7295147150e1b49fa42649
+  metadata.gz: ddef939463b55a174ccb6f44f90ad8f28f9f8bdc
+  data.tar.gz: b80c4c9a95e03b2fc61bd378f1e405a26a633513
 SHA512:
-  metadata.gz: a91d0f3867c8ceda6e96d134bacffaf22f50c6332077b177060cd5a6b03d77e22ce7e81af7770ed18e989e6358324cf54e8cb2bdb0124bd3ea6567994cac8144
-  data.tar.gz: 07322eb0b6961b86c92ce5b3798764847695479473d043167160cb5c9ca07783f11a9dee2cc33eb96d872719abe042225edbb5cbe16cada66623c93837861a34
+  metadata.gz: 195e87944d7a7d3d447d7526ee58459c091381a5c7c25f78ac7c1768fc9857f1f3106f8b011554f3d232e6b7dd36f47a24cf402bc861912751d3abfc2317f79a
+  data.tar.gz: e19354cfca2219c5d2765e268a2d8a93ffd243168b50bd46b87e8b023cfc976d74f52631926daa917f33a4674b9be75d254be1f745f2f900084bd9a1a9570b95

data/README.md

@@ -5,6 +5,35 @@
 
 Map-Reduce where both the mappers and the reducer run inside Docker containers.
 
+## Development Environment
+
+`contained-mr` requires access to a Docker daemon. The easiest way to
+bring up a development setup is to install
+[Docker Machine](https://github.com/docker/machine) and
+[VirtualBox](https://www.virtualbox.org/).
+
+The commands below install the prerequisites on OSX using
+[Homebrew](http://brew.sh/).
+
+```bash
+brew install brew-cask docker docker-machine
+brew cask install virtualbox
+```
+
+Create a Docker VM. This is a one-time setup.
+
+```bash
+docker-machine create --driver virtualbox dev
+```
+
+Set up the local environment to point to the Docker daemon in the VM. This must
+be executed in every shell where `contained-mr` is used.
+
+```bash
+eval "$(docker-machine env dev)"
+```
+
+
 ## Contributing to contained_mr
 
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/contained_mr.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: contained_mr 0.2.0 ruby lib
+# stub: contained_mr 0.3.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "contained_mr"
-  s.version = "0.2.0"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Victor Costan"]
-  s.date = "2015-09-17"
+  s.date = "2015-10-09"
   s.description = "Plumbing for running mappers and reducers inside Docker containers"
   s.email = "victor@costan.us"
   s.extra_rdoc_files = [
@@ -67,7 +67,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/pwnall/contained_mr"
   s.licenses = ["MIT"]
-  s.rubygems_version = "2.4.5"
+  s.rubygems_version = "2.4.5.1"
   s.summary = "Map-Reduce with Docker containers"
 
   if s.respond_to? :specification_version then

data/lib/contained_mr/job.rb

@@ -9,7 +9,7 @@ require 'docker'
 class ContainedMr::Job
   include ContainedMr::JobLogic
 
-  # @see {ContainedMr::TemplateLogic#create_job}
+  # @see {ContainedMr::TemplateLogic#new_job}
   def initialize(template, id, json_options)
     @template = template
     @id = id

data/lib/contained_mr/job_logic.rb

@@ -55,13 +55,17 @@ module ContainedMr::JobLogic
       { "Name" => k.to_s, "Soft" => v, "Hard" => v }
     end
 
+    env = @template.mapper_env i
+    env.push "affinity:image==#{mapper_image_tag}"
+
     {
       'name' => "#{@name_prefix}_mapper.#{@id}.#{i}",
-      'Image' => @mapper_image_id,
+      'Image' => mapper_image_tag,
       'Hostname' => "#{i}.mapper", 'Domainname' => '',
       'Labels' => { 'contained_mr.ctl' => @name_prefix },
-      'Env' => @template.mapper_env(i), 'Ulimits' => ulimits,
+      'Env' => env, 'Ulimits' => ulimits,
       'NetworkDisabled' => true, 'ExposedPorts' => {},
+      'HostConfig' => container_host_config(@mapper_options),
     }
   end
 
@@ -71,15 +75,45 @@ module ContainedMr::JobLogic
       { "Name" => k.to_s, "Soft" => v, "Hard" => v }
     end
 
+    env = @template.reducer_env
+    env.push "affinity:image==#{reducer_image_tag}"
+
     {
       'name' => "#{@name_prefix}_reducer.#{@id}",
-      'Image' => @reducer_image_id,
+      'Image' => reducer_image_tag,
       'Hostname' => 'reducer', 'Domainname' => '',
       'Labels' => { 'contained_mr.ctl' => @name_prefix },
-      'Env' => @template.reducer_env, 'Ulimits' => ulimits,
+      'Env' => env, 'Ulimits' => ulimits,
       'NetworkDisabled' => true, 'ExposedPorts' => {},
+      'HostConfig' => container_host_config(@reducer_options),
+    }
+  end
+
+  # Computes the value of the HostConfig key in container creation params.
+  #
+  # @param {Hash<Symbol, Object>} job_section the "mapper" or "reducer" section
+  #   in the options
+  # @return {Hash<String, Object>} a container's HostConfig params
+  def container_host_config(job_section)
+    ram_bytes = (job_section[:ram] * 1048576).to_i
+    if job_section[:swap] == 0
+      swap_bytes = -1
+    else
+      swap_bytes = (job_section[:swap] * 1048576).to_i + ram_bytes
+    end
+
+    # NOTE: The value below is 1 second, in microsecodns. This is the maximum
+    #       value, and it minimizes scheduling overheads, at the expense of
+    #       precision.
+    cpu_period = 1_000_000
+
+    {
+      'Memory' => ram_bytes, 'MemorySwap' => swap_bytes,
+      'CpuShares' => (job_section[:vcpus] * cpu_period).to_i,
+      'CpuPeriod' => cpu_period
     }
   end
+  private :container_host_config
 
   # Reads in JSON options and sets defaults.
   def parse_options(json_options)
@@ -87,9 +121,11 @@ module ContainedMr::JobLogic
     mapper_ulimits = mapper['ulimits'] || {}
     @mapper_options = {
       wait_time: mapper['wait_time'] || 60,
+      vcpus: mapper['vcpus'] || 1,  # logical processors
+      ram: mapper['ram'] || 512,  # megabytes
+      swap: mapper['swap'] || 0,  # megabytes
       ulimits: {
         cpu: mapper_ulimits['cpu'] || 60,  # seconds
-        rss: mapper_ulimits['rss'] || 500_000,  # pages
       }
     }
 
@@ -97,9 +133,11 @@ module ContainedMr::JobLogic
     reducer_ulimits = reducer['ulimits'] || {}
     @reducer_options = {
       wait_time: reducer['wait_time'] || 60,
+      vcpus: reducer['vcpus'] || 1,  # logical processors
+      ram: reducer['ram'] || 512,  # megabytes
+      swap: reducer['swap'] || 0,  # megabytes
       ulimits: {
         cpu: reducer_ulimits['cpu'] || 60,
-        rss: reducer_ulimits['rss'] || 500_000,
       }
     }
   end

data/lib/contained_mr/template_logic.rb

@@ -69,7 +69,7 @@ module ContainedMr::TemplateLogic
   # @private common code from mapper_dockerfile and reducer_dockerfile
   def job_dockerfile(job_definition, input_source)
     <<DOCKER_END
-FROM #{@image_id}
+FROM #{image_tag}
 COPY #{input_source} #{job_definition['input'] || '/input'}
 WORKDIR #{job_definition['chdir'] || '/'}
 ENTRYPOINT #{JSON.dump(job_definition['cmd'] || ['/bin/sh'])}

data/test/test_job_logic.rb

@@ -9,6 +9,14 @@ class TestJobLogic < MiniTest::Test
         JSON.load(File.read('testdata/job.hello'))
   end
 
+  def test_mapper_image_tag
+    assert_equal 'contained_mrtests/mapper.testjob', @job.mapper_image_tag
+  end
+
+  def test_reducer_image_tag
+    assert_equal 'contained_mrtests/reducer.testjob', @job.reducer_image_tag
+  end
+
   def test_mapper_container_options
     assert_equal @template, @job.template
     assert_equal 'contained_mrtests', @job.name_prefix
@@ -19,16 +27,22 @@ class TestJobLogic < MiniTest::Test
 
     golden = {
       'name' => 'contained_mrtests_mapper.testjob.2',
-      'Image' => @job.mapper_image_id,
+      'Image' => 'contained_mrtests/mapper.testjob',
       'Hostname' => '2.mapper',
       'Domainname' => '',
       'Labels' => { 'contained_mr.ctl' => 'contained_mrtests' },
-      'Env' => [ 'ITEM=2', 'ITEMS=3' ],
+      'Env' => [ 'ITEM=2', 'ITEMS=3',
+                 'affinity:image==contained_mrtests/mapper.testjob' ],
       'Ulimits' => [
         { 'Name' => 'cpu', 'Hard' => 3, 'Soft' => 3 },
-        { 'Name' => 'rss', 'Hard' => 1000000, 'Soft' => 1000000 },
       ],
       'NetworkDisabled' => true, 'ExposedPorts' => {},
+      'HostConfig' => {
+        'Memory' => 256.5 * 1024 * 1024,
+        'MemorySwap' => (256.5 + 64) * 1024 * 1024,
+        'CpuShares' => 1500000,
+        'CpuPeriod' => 1000000,
+      },
     }
     assert_equal golden, @job.mapper_container_options(2)
   end
@@ -36,16 +50,22 @@ class TestJobLogic < MiniTest::Test
   def test_reducer_container_options
     golden = {
       'name' => 'contained_mrtests_reducer.testjob',
-      'Image' => @job.mapper_image_id,
+      'Image' => 'contained_mrtests/reducer.testjob',
       'Hostname' => 'reducer',
       'Domainname' => '',
       'Labels' => { 'contained_mr.ctl' => 'contained_mrtests' },
-      'Env' => [ 'ITEMS=3' ],
+      'Env' => [ 'ITEMS=3',
+                 'affinity:image==contained_mrtests/reducer.testjob' ],
       'Ulimits' => [
         { 'Name' => 'cpu', 'Hard' => 2, 'Soft' => 2 },
-        { 'Name' => 'rss', 'Hard' => 100000, 'Soft' => 100000 },
       ],
       'NetworkDisabled' => true, 'ExposedPorts' => {},
+      'HostConfig' => {
+        'Memory' => 768.5 * 1024 * 1024,
+        'MemorySwap' => -1,
+        'CpuShares' => 500000,
+        'CpuPeriod' => 1000000,
+      },
     }
     assert_equal golden, @job.reducer_container_options
   end

data/test/test_mock_runner.rb

@@ -14,6 +14,12 @@ class TestMockRunner < MiniTest::Test
         { 'Name' => 'rss', 'Hard' => 1000000, 'Soft' => 1000000 },
       ],
       'NetworkDisabled' => true, 'ExposedPorts' => {},
+      'HostConfig' => {
+        'Memory' => 256.5 * 1024 * 1024,
+        'MemorySwap' => (256.5 + 64) * 1024 * 1024,
+        'CpuShares' => 1500000,
+        'CpuPeriod' => 1000000,
+      },
     }
     @runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                             '/usr/mrd/map-output'

data/test/test_runner_logic.rb

@@ -11,9 +11,14 @@ class TestRunnerLogic < MiniTest::Test
       'Env' => [ 'ITEM=2', 'ITEMS=3' ],
       'Ulimits' => [
         { 'Name' => 'cpu', 'Hard' => 3, 'Soft' => 3 },
-        { 'Name' => 'rss', 'Hard' => 1000000, 'Soft' => 1000000 },
       ],
       'NetworkDisabled' => true, 'ExposedPorts' => {},
+      'HostConfig' => {
+        'Memory' => 256.5 * 1024 * 1024,
+        'MemorySwap' => (256.5 + 64) * 1024 * 1024,
+        'CpuShares' => 1500000,
+        'CpuPeriod' => 1000000,
+      },
     }
     @runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                             '/usr/mrd/map-output'

data/test/test_template_logic.rb

@@ -9,13 +9,11 @@ class TestTemplateLogic < MiniTest::Test
 
   def test_mapper_dockerfile
     golden = File.read 'testdata/Dockerfile.hello.mapper'
-    golden.sub! 'contained_mrtests/base.hello', 'mock-template-image-id'
     assert_equal golden, @template.mapper_dockerfile, 'mapper Dockerfile'
   end
 
   def test_reducer_dockerfile
     golden = File.read 'testdata/Dockerfile.hello.reducer'
-    golden.sub! 'contained_mrtests/base.hello', 'mock-template-image-id'
     assert_equal golden, @template.reducer_dockerfile, 'reducer Dockerfile'
   end
 

data/testdata/job.hello

@@ -1,10 +1,16 @@
 {
   "mapper": {
     "wait_time": 2.5,
-    "ulimits": { "cpu": 3, "rss": 1000000 }
+    "vcpus": 1.5,
+    "ram": 256.5,
+    "swap": 64,
+    "ulimits": { "cpu": 3 }
   },
   "reducer": {
     "wait_time": 2,
-    "ulimits": { "cpu": 2, "rss": 100000 }
+    "vcpus": 0.5,
+    "ram": 768.5,
+    "swap": 0,
+    "ulimits": { "cpu": 2 }
   }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contained_mr
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Victor Costan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-17 00:00:00.000000000 Z
+date: 2015-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api
@@ -209,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Map-Reduce with Docker containers