contained_mr 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/VERSION +1 -1
- data/contained_mr.gemspec +3 -3
- data/lib/contained_mr/cleaner.rb +15 -5
- data/lib/contained_mr/runner.rb +5 -1
- data/test/test_cleaner.rb +59 -2
- data/test/test_runner.rb +35 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a2802a380e19e314a40493959bace7208a0a8109
|
4
|
+
data.tar.gz: fb912e50f9e146624364edda1272faeff971f133
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82d975a57ef901d4909d8dce6f92d240725fb3ad09d8c3e8f0614c51d98df0cdfbc6c7435bffeb1f734b40025d9487bb89f7e2c40c2e41fb9433d48df2b6238a
|
7
|
+
data.tar.gz: 6f034abfe4bf32dd2ab17ac17c89eab1c7b6d3f1cf2e4bd4296288c638421ec89f3d97cf8252836ef88d1473fe8e4bf166399f1cbf6f51a56816d2597db19981
|
data/README.md
CHANGED
@@ -23,7 +23,7 @@ brew cask install virtualbox
|
|
23
23
|
Create a Docker VM. This is a one-time setup.
|
24
24
|
|
25
25
|
```bash
|
26
|
-
docker-machine create --driver virtualbox dev
|
26
|
+
docker-machine create --driver virtualbox --engine-storage-driver overlay dev
|
27
27
|
```
|
28
28
|
|
29
29
|
Set up the local environment to point to the Docker daemon in the VM. This must
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.4.
|
1
|
+
0.4.1
|
data/contained_mr.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: contained_mr 0.4.
|
5
|
+
# stub: contained_mr 0.4.1 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "contained_mr"
|
9
|
-
s.version = "0.4.
|
9
|
+
s.version = "0.4.1"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Victor Costan"]
|
14
|
-
s.date = "2015-10-
|
14
|
+
s.date = "2015-10-27"
|
15
15
|
s.description = "Plumbing for running mappers and reducers inside Docker containers"
|
16
16
|
s.email = "victor@costan.us"
|
17
17
|
s.extra_rdoc_files = [
|
data/lib/contained_mr/cleaner.rb
CHANGED
@@ -14,23 +14,32 @@ class ContainedMr::Cleaner
|
|
14
14
|
end
|
15
15
|
|
16
16
|
# Removes all images and containers matching this cleaner's name prefix.
|
17
|
+
#
|
18
|
+
# @return {Array<Docker::Container, Docker::Image>} the removed objects
|
17
19
|
def destroy_all!
|
18
|
-
destroy_all_containers!
|
19
|
-
destroy_all_images!
|
20
|
+
containers = destroy_all_containers!
|
21
|
+
images = destroy_all_images!
|
22
|
+
containers + images
|
20
23
|
end
|
21
24
|
|
25
|
+
# @return {Array<Docker::Container>} the removed containers
|
22
26
|
def destroy_all_containers!
|
23
27
|
containers = Docker::Container.all all: true,
|
24
28
|
filters: container_filters.to_json
|
25
29
|
containers.each do |container|
|
26
|
-
|
30
|
+
begin
|
31
|
+
container.delete force: false, volumes: true
|
32
|
+
rescue Docker::Error::NotFoundError
|
33
|
+
# Workaround for https://github.com/docker/docker/issues/14474
|
34
|
+
end
|
27
35
|
end
|
28
36
|
end
|
29
|
-
private :destroy_all_containers!
|
30
37
|
|
38
|
+
# @return {Array<Docker::Image>} the removed images
|
31
39
|
def destroy_all_images!
|
32
40
|
tag_prefix = "#{@name_prefix}/"
|
33
41
|
images = Docker::Image.all
|
42
|
+
deleted_images = []
|
34
43
|
images.each do |image|
|
35
44
|
next unless image_tags = image.info['RepoTags']
|
36
45
|
image_tags.each do |image_tag|
|
@@ -38,10 +47,11 @@ class ContainedMr::Cleaner
|
|
38
47
|
# HACK(pwnall): Trick docker-api into issuing a DELETE request by tag.
|
39
48
|
tag_image = Docker::Image.new Docker.connection, 'id' => image_tag
|
40
49
|
tag_image.delete
|
50
|
+
deleted_images << image
|
41
51
|
end
|
42
52
|
end
|
53
|
+
deleted_images
|
43
54
|
end
|
44
|
-
private :destroy_all_images!
|
45
55
|
|
46
56
|
# @return { Hash<Symbol, Array<String>> } filters used to identify Docker
|
47
57
|
# containers started by this controller
|
data/lib/contained_mr/runner.rb
CHANGED
@@ -50,7 +50,11 @@ class ContainedMr::Runner
|
|
50
50
|
def destroy!(container = nil)
|
51
51
|
unless @container_id.nil?
|
52
52
|
container ||= Docker::Container.get @container_id
|
53
|
-
|
53
|
+
begin
|
54
|
+
container.delete force: true
|
55
|
+
rescue Docker::Error::NotFoundError
|
56
|
+
# Workaround for https://github.com/docker/docker/issues/14474
|
57
|
+
end
|
54
58
|
@container_id = nil
|
55
59
|
end
|
56
60
|
self
|
data/test/test_cleaner.rb
CHANGED
@@ -7,12 +7,30 @@ class TestCleaner < MiniTest::Test
|
|
7
7
|
@job = @template.new_job 'testjob',
|
8
8
|
JSON.load(File.read('testdata/job.hello'))
|
9
9
|
@job.build_mapper_image File.read('testdata/input.hello')
|
10
|
+
@runner = ContainedMr::Runner.new @job.mapper_container_options(2), 2.5,
|
11
|
+
@template.mapper_output_path
|
12
|
+
class <<@runner
|
13
|
+
def destroy!(*args)
|
14
|
+
self
|
15
|
+
end
|
16
|
+
end
|
17
|
+
@runner.perform
|
10
18
|
|
11
19
|
@cleaner = ContainedMr::Cleaner.new 'contained_mrtests'
|
12
20
|
end
|
13
21
|
|
14
|
-
def
|
22
|
+
def teardown
|
23
|
+
Docker::Container.any_instance.unstub :delete
|
15
24
|
@cleaner.destroy_all!
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_destroy_all
|
28
|
+
objects = @cleaner.destroy_all!
|
29
|
+
assert_equal 3, objects.length
|
30
|
+
assert_kind_of Docker::Container, objects[0]
|
31
|
+
assert_kind_of Docker::Image, objects[1]
|
32
|
+
assert_kind_of Docker::Image, objects[2]
|
33
|
+
|
16
34
|
assert_raises Docker::Error::NotFoundError do
|
17
35
|
Docker::Image.get @job.mapper_image_tag
|
18
36
|
end
|
@@ -27,7 +45,24 @@ class TestCleaner < MiniTest::Test
|
|
27
45
|
job2 = template2.new_job 'testjob2',
|
28
46
|
JSON.load(File.read('testdata/job.hello'))
|
29
47
|
job2.build_mapper_image File.read('testdata/input.hello')
|
30
|
-
|
48
|
+
runner2 = ContainedMr::Runner.new job2.mapper_container_options(2), 2.5,
|
49
|
+
template2.mapper_output_path
|
50
|
+
class <<runner2
|
51
|
+
def destroy!(*args)
|
52
|
+
self
|
53
|
+
end
|
54
|
+
end
|
55
|
+
runner2.perform
|
56
|
+
|
57
|
+
objects = @cleaner.destroy_all!
|
58
|
+
assert_equal 6, objects.length
|
59
|
+
assert_kind_of Docker::Container, objects[0]
|
60
|
+
assert_kind_of Docker::Container, objects[1]
|
61
|
+
assert_kind_of Docker::Image, objects[2]
|
62
|
+
assert_kind_of Docker::Image, objects[3]
|
63
|
+
assert_kind_of Docker::Image, objects[4]
|
64
|
+
assert_kind_of Docker::Image, objects[5]
|
65
|
+
|
31
66
|
assert_raises Docker::Error::NotFoundError do
|
32
67
|
Docker::Image.get job2.mapper_image_tag
|
33
68
|
end
|
@@ -41,4 +76,26 @@ class TestCleaner < MiniTest::Test
|
|
41
76
|
Docker::Image.get @template.image_tag
|
42
77
|
end
|
43
78
|
end
|
79
|
+
|
80
|
+
def test_destroy_all_containers_with_duplicates_and_exceptions
|
81
|
+
template2 = ContainedMr.new_template 'contained_mrtests', 'hello2',
|
82
|
+
StringIO.new(File.binread('testdata/hello.zip'))
|
83
|
+
job2 = template2.new_job 'testjob2',
|
84
|
+
JSON.load(File.read('testdata/job.hello'))
|
85
|
+
job2.build_mapper_image File.read('testdata/input.hello')
|
86
|
+
runner2 = ContainedMr::Runner.new job2.mapper_container_options(2), 2.5,
|
87
|
+
template2.mapper_output_path
|
88
|
+
class <<runner2
|
89
|
+
def destroy!(*args)
|
90
|
+
self
|
91
|
+
end
|
92
|
+
end
|
93
|
+
runner2.perform
|
94
|
+
|
95
|
+
Docker::Container.any_instance.expects(:delete).twice.
|
96
|
+
raises Docker::Error::NotFoundError
|
97
|
+
containers = @cleaner.destroy_all_containers!
|
98
|
+
assert_equal 2, containers.length
|
99
|
+
containers.each { |c| assert_kind_of Docker::Container, c }
|
100
|
+
end
|
44
101
|
end
|
data/test/test_runner.rb
CHANGED
@@ -68,4 +68,39 @@ class TestRunner < MiniTest::Test
|
|
68
68
|
@template.mapper_output_path
|
69
69
|
assert_equal runner, runner.destroy!
|
70
70
|
end
|
71
|
+
|
72
|
+
def test_destroy_invalid_container
|
73
|
+
runner = ContainedMr::Runner.new @job.mapper_container_options(2), 2.5,
|
74
|
+
@template.mapper_output_path
|
75
|
+
|
76
|
+
# HACK(pwnall): This is a dirty way of getting NotFoundError, which is also
|
77
|
+
# what the docker gem throws in docker#14474.
|
78
|
+
class <<runner
|
79
|
+
alias_method :old_destroy, :destroy!
|
80
|
+
def destroy!(container)
|
81
|
+
old_container_id = @container_id
|
82
|
+
old_destroy container
|
83
|
+
if container
|
84
|
+
@container_id = old_container_id
|
85
|
+
old_destroy container
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
assert_equal runner, runner.perform
|
91
|
+
|
92
|
+
assert_equal nil, runner.container_id, 'container still running'
|
93
|
+
assert_operator runner.ended_at - runner.started_at, :<, 1, 'running time'
|
94
|
+
assert_equal 0, runner.status_code, 'status code'
|
95
|
+
assert_equal false, runner.timed_out, 'timed out'
|
96
|
+
assert_equal "2 3\n", runner.stderr, 'Stderr: $ITEM + $ITEMS'
|
97
|
+
assert_equal "2\nmapper input file\nHello world!\n", runner.stdout,
|
98
|
+
'Stdout: $ITEM + mapper input file + data file'
|
99
|
+
assert_equal "2\n", runner.output, 'Output: ITEM env variable'
|
100
|
+
|
101
|
+
assert_equal runner.ended_at - runner.started_at,
|
102
|
+
runner.json_file[:ran_for]
|
103
|
+
assert_equal 0, runner.json_file[:exit_code]
|
104
|
+
assert_equal false, runner.json_file[:timed_out]
|
105
|
+
end
|
71
106
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: contained_mr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Victor Costan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-10-
|
11
|
+
date: 2015-10-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: docker-api
|