contained_mr 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aa36265b30cbc358e74b368ebdf5cf9a90c6ebaa
-  data.tar.gz: f10177d6247c5be5e24c7c47cc72e1511f1805d3
+  metadata.gz: e185d4610f9518f8f95924cd6569f8f62bdd309d
+  data.tar.gz: 9dc30131aa6475f07b3e5042e43877dbab91d0a3
 SHA512:
-  metadata.gz: be014222e385e4b87ac25e5d4e979bd1931b1d6f3ffc304543a11fab3857fb82030eca49d68818c8d3ffe33366518d7b178e77af96c546459afd2e3200afe4d5
-  data.tar.gz: 9a31100c13d897147cd0ca6bef0dc53fd5109d701b0dc4c6c3a5bd469a246d7385e5fe71d33a9e93124c679bc82ef97424048a430159e8f41c79c7ad2063990d
+  metadata.gz: 597919494dce5409863b1ecd5d8ea9b75870f6705d945e74822995501184b6a0138fdd730a6c04e2a0bbe1bb7c85bcb3028f5a1c6291ec2bd0284bc01d06dfc3
+  data.tar.gz: c0936fd70b787f76aa222186ee6e915b9d400f0abb036af0c09a038c33a6049e4cf11d4435ab3c0dea6291a5fae6feb13a0b7eb6c647e4fa24e4a006f49add3d

data/VERSION

@@ -1 +1 @@
-0.3.4
+0.4.0

data/contained_mr.gemspec

@@ -2,16 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: contained_mr 0.3.4 ruby lib
+# stub: contained_mr 0.4.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "contained_mr"
-  s.version = "0.3.4"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib"]
   s.authors = ["Victor Costan"]
-  s.date = "2015-10-13"
+  s.date = "2015-10-17"
   s.description = "Plumbing for running mappers and reducers inside Docker containers"
   s.email = "victor@costan.us"
   s.extra_rdoc_files = [

data/lib/contained_mr/job_logic.rb

@@ -96,11 +96,7 @@ module ContainedMr::JobLogic
   # @return {Hash<String, Object>} a container's HostConfig params
   def container_host_config(job_section)
     ram_bytes = (job_section[:ram] * 1048576).to_i
-    if job_section[:swap] == 0
-      swap_bytes = -1
-    else
-      swap_bytes = (job_section[:swap] * 1048576).to_i + ram_bytes
-    end
+    swap_bytes = (job_section[:swap] * 1048576).to_i + ram_bytes
 
     # NOTE: The value below is 1 second, in microsecodns. This is the maximum
     #       value, and it minimizes scheduling overheads, at the expense of
@@ -109,8 +105,16 @@ module ContainedMr::JobLogic
 
     {
       'Memory' => ram_bytes, 'MemorySwap' => swap_bytes,
+      'MemorySwappiness' => 0,
       'CpuPeriod' => cpu_period,
       'CpuQuota' => (job_section[:vcpus] * cpu_period).to_i,
+      'LogConfig' => {
+        'Type' => 'json-file',
+        'Config' => {
+          'max-size' => (job_section[:logs] * 1048576).to_i.to_s,
+          'max-file' => '1',
+        },
+      },
     }
   end
   private :container_host_config
@@ -122,8 +126,9 @@ module ContainedMr::JobLogic
     @mapper_options = {
       wait_time: mapper['wait_time'] || 60,
       vcpus: mapper['vcpus'] || 1,  # logical processors
-      ram: mapper['ram'] || 512,  # megabytes
-      swap: mapper['swap'] || 0,  # megabytes
+      ram: mapper['ram'] || 512,    # megabytes
+      swap: mapper['swap'] || 0,    # megabytes
+      logs: mapper['logs'] || 64,   # megabytes
       ulimits: {
         cpu: mapper_ulimits['cpu'] || 60,  # seconds
       }
@@ -134,8 +139,9 @@ module ContainedMr::JobLogic
     @reducer_options = {
       wait_time: reducer['wait_time'] || 60,
       vcpus: reducer['vcpus'] || 1,  # logical processors
-      ram: reducer['ram'] || 512,  # megabytes
-      swap: reducer['swap'] || 0,  # megabytes
+      ram: reducer['ram'] || 512,    # megabytes
+      swap: reducer['swap'] || 0,    # megabytes
+      logs: reducer['logs'] || 64,   # megabytes
       ulimits: {
         cpu: reducer_ulimits['cpu'] || 60,
       }

data/lib/contained_mr/mock/runner.rb

@@ -101,7 +101,6 @@ class ContainedMr::Mock::Runner
     return nil unless memory = host_config['Memory']
     return nil unless memory_swap = host_config['MemorySwap']
 
-    return 0 if memory_swap == -1
     (memory_swap - memory) / (1024 * 1024).to_f
   end
 
@@ -116,4 +115,16 @@ class ContainedMr::Mock::Runner
 
     quota / period.to_f
   end
+
+  # Convenience method for looking up the log size limit in container options.
+  #
+  # @return {Number} the container's log limit, in megabytes
+  def _logs
+    return nil unless host_config = @_container_options['HostConfig']
+    return nil unless log_config = host_config['LogConfig']
+    return nil unless config = log_config['Config']
+    return nil unless max_size = config['max-size']
+
+    max_size.to_i / (1024 * 1024).to_f
+  end
 end

data/test/test_job_logic.rb

@@ -40,8 +40,16 @@ class TestJobLogic < MiniTest::Test
       'HostConfig' => {
         'Memory' => 256.5 * 1024 * 1024,
         'MemorySwap' => (256.5 + 64) * 1024 * 1024,
+        'MemorySwappiness' => 0,
         'CpuQuota' => 1500000,
         'CpuPeriod' => 1000000,
+        'LogConfig' => {
+          'Type' => 'json-file',
+          'Config' => {
+            'max-size' => '4718592',
+            'max-file' => '1',
+          },
+        },
       },
     }
     assert_equal golden, @job.mapper_container_options(2)
@@ -62,9 +70,17 @@ class TestJobLogic < MiniTest::Test
       'NetworkDisabled' => true, 'ExposedPorts' => {},
       'HostConfig' => {
         'Memory' => 768.5 * 1024 * 1024,
-        'MemorySwap' => -1,
+        'MemorySwap' => 768.5 * 1024 * 1024,
+        'MemorySwappiness' => 0,
         'CpuQuota' => 500000,
         'CpuPeriod' => 1000000,
+        'LogConfig' => {
+          'Type' => 'json-file',
+          'Config' => {
+            'max-size' => '6815744',
+            'max-file' => '1',
+          },
+        },
       },
     }
     assert_equal golden, @job.reducer_container_options

data/test/test_mock_runner.rb

@@ -17,8 +17,16 @@ class TestMockRunner < MiniTest::Test
       'HostConfig' => {
         'Memory' => 256.5 * 1024 * 1024,
         'MemorySwap' => (256.5 + 64) * 1024 * 1024,
+        'MemorySwappiness' => 0,
         'CpuQuota' => 1500000,
         'CpuPeriod' => 1000000,
+        'LogConfig' => {
+          'Type' => 'json-file',
+          'Config' => {
+            'max-size' => '4718592',
+            'max-file' => '1',
+          },
+        },
       },
     }
     @runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
@@ -103,63 +111,100 @@ class TestMockRunner < MiniTest::Test
     assert_equal nil, runner._ulimit('rss')
   end
 
-  def test_memory_cpu_limits
+  def test_resources
     assert_equal 256.5, @runner._ram_limit
     assert_equal 64, @runner._swap_limit
     assert_equal 1.5, @runner._vcpus
+    assert_equal 4.5, @runner._logs
   end
 
-  def test_memory_cpu_limits_without_host_config
+  def test_resources_without_host_config
     @container_options.delete 'HostConfig'
     runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                            '/usr/mrd/map-output'
     assert_equal nil, runner._ram_limit
     assert_equal nil, runner._swap_limit
     assert_equal nil, runner._vcpus
+    assert_equal nil, runner._logs
   end
 
-  def test_memory_cpu_limits_without_host_config_memory
+  def test_resources_without_host_config_memory
     @container_options['HostConfig'].delete 'Memory'
     runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                            '/usr/mrd/map-output'
     assert_equal nil, runner._ram_limit
     assert_equal nil, runner._swap_limit
     assert_equal 1.5, runner._vcpus
+    assert_equal 4.5, runner._logs
   end
 
-  def test_memory_cpu_limits_without_host_config_memory_swap
+  def test_resources_without_host_config_memory_swap
     @container_options['HostConfig'].delete 'MemorySwap'
     runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal nil, runner._swap_limit
     assert_equal 1.5, runner._vcpus
+    assert_equal 4.5, runner._logs
   end
 
-  def test_memory_cpu_limits_without_host_config_cpu_shares
+  def test_resources_without_host_config_cpu_shares
     @container_options['HostConfig'].delete 'CpuQuota'
     runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 64, runner._swap_limit
     assert_equal nil, runner._vcpus
+    assert_equal 4.5, runner._logs
   end
 
-  def test_memory_cpu_limits_without_host_config_cpu_period
+  def test_resources_without_host_config_cpu_period
     @container_options['HostConfig'].delete 'CpuPeriod'
     runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 64, runner._swap_limit
     assert_equal nil, runner._vcpus
+    assert_equal 4.5, runner._logs
   end
 
-  def test_memory_limits_without_swap
-    @container_options['HostConfig']['MemorySwap'] = -1
+  def test_resources_without_host_config_log_config
+    @container_options['HostConfig'].delete 'LogConfig'
+    runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
+                                           '/usr/mrd/map-output'
+    assert_equal 256.5, runner._ram_limit
+    assert_equal 64, runner._swap_limit
+    assert_equal 1.5, runner._vcpus
+    assert_equal nil, runner._logs
+  end
+
+  def test_resources_without_host_config_log_config_config
+    @container_options['HostConfig']['LogConfig'].delete 'Config'
+    runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
+                                           '/usr/mrd/map-output'
+    assert_equal 256.5, runner._ram_limit
+    assert_equal 64, runner._swap_limit
+    assert_equal 1.5, runner._vcpus
+    assert_equal nil, runner._logs
+  end
+
+  def test_resources_without_host_config_log_config_config_max_size
+    @container_options['HostConfig']['LogConfig']['Config'].delete 'max-size'
+    runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
+                                           '/usr/mrd/map-output'
+    assert_equal 256.5, runner._ram_limit
+    assert_equal 64, runner._swap_limit
+    assert_equal 1.5, runner._vcpus
+    assert_equal nil, runner._logs
+  end
+
+  def test_resources_with_zero_swap
+    @container_options['HostConfig']['MemorySwap'] = 256.5 * 1024 * 1024
     runner = ContainedMr::Mock::Runner.new @container_options, 2.5,
                                            '/usr/mrd/map-output'
     assert_equal 256.5, runner._ram_limit
     assert_equal 0, runner._swap_limit
     assert_equal 1.5, runner._vcpus
+    assert_equal 4.5, runner._logs
   end
 end

data/test/test_runner_logic.rb

@@ -16,8 +16,16 @@ class TestRunnerLogic < MiniTest::Test
       'HostConfig' => {
         'Memory' => 256.5 * 1024 * 1024,
         'MemorySwap' => (256.5 + 64) * 1024 * 1024,
+        'MemorySwappiness' => 0,
         'CpuQuota' => 1500000,
         'CpuPeriod' => 1000000,
+        'LogConfig' => {
+          'Type' => 'json-file',
+          'Config' => {
+            'max-size' => '4718592',
+            'max-file' => '1',
+          },
+        },
       },
     }
     @runner = ContainedMr::Mock::Runner.new @container_options, 2.5,

data/testdata/job.hello

@@ -4,6 +4,7 @@
     "vcpus": 1.5,
     "ram": 256.5,
     "swap": 64,
+    "logs": 4.5,
     "ulimits": { "cpu": 3 }
   },
   "reducer": {
@@ -11,6 +12,7 @@
     "vcpus": 0.5,
     "ram": 768.5,
     "swap": 0,
+    "logs": 6.5,
     "ulimits": { "cpu": 2 }
   }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contained_mr
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Victor Costan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-13 00:00:00.000000000 Z
+date: 2015-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: docker-api