map_reduce 0.0.1.alpha → 0.0.1.alpha2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/map_reduce/version.rb +1 -1
- data/lib/map_reduce/worker.rb +1 -1
- data/spec/map_reduce/map_reduce_spec.rb +81 -39
- metadata +3 -3
data/lib/map_reduce/version.rb
CHANGED
data/lib/map_reduce/worker.rb
CHANGED
@@ -1,34 +1,36 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
3
|
describe "MapReduce stack" do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
describe "single master" do
|
5
|
+
before do
|
6
|
+
@pid = fork do
|
7
|
+
master = MapReduce::Master.new
|
8
|
+
master.run
|
9
|
+
end
|
8
10
|
end
|
9
|
-
end
|
10
11
|
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
after do
|
13
|
+
Process.kill "TERM", @pid
|
14
|
+
end
|
15
|
+
|
16
|
+
it "should map and reduce some data in CB mode" do
|
17
|
+
EM.run do
|
18
|
+
data = {}
|
19
|
+
worker = MapReduce::Worker.new
|
20
|
+
worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',') do
|
21
|
+
worker.map("Alex", ["Madonna", "Lady Gaga"] * ',') do
|
22
|
+
worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',') do
|
23
|
+
worker.map_finished do
|
24
|
+
worker.reduce do |key, values|
|
25
|
+
if key
|
26
|
+
data[key] = values
|
27
|
+
else
|
28
|
+
data.size.must_equal 2
|
29
|
+
data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
|
30
|
+
data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
|
14
31
|
|
15
|
-
|
16
|
-
|
17
|
-
data = {}
|
18
|
-
worker = MapReduce::Worker.new
|
19
|
-
worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',') do
|
20
|
-
worker.map("Alex", ["Madonna", "Lady Gaga"] * ',') do
|
21
|
-
worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',') do
|
22
|
-
worker.map_finished do
|
23
|
-
worker.reduce do |key, values|
|
24
|
-
if key
|
25
|
-
data[key] = values
|
26
|
-
else
|
27
|
-
data.size.must_equal 2
|
28
|
-
data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
|
29
|
-
data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
|
30
|
-
|
31
|
-
EM.stop
|
32
|
+
EM.stop
|
33
|
+
end
|
32
34
|
end
|
33
35
|
end
|
34
36
|
end
|
@@ -36,24 +38,64 @@ describe "MapReduce stack" do
|
|
36
38
|
end
|
37
39
|
end
|
38
40
|
end
|
41
|
+
|
42
|
+
it "should map and reduce some data in SYNC mode" do
|
43
|
+
EM.synchrony do
|
44
|
+
data = {}
|
45
|
+
worker = MapReduce::Worker.new type: :sync
|
46
|
+
worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
|
47
|
+
worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
|
48
|
+
worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
|
49
|
+
worker.map_finished
|
50
|
+
worker.reduce do |key, values|
|
51
|
+
data[key] = values if key
|
52
|
+
end
|
53
|
+
data.size.must_equal 2
|
54
|
+
data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
|
55
|
+
data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
|
56
|
+
|
57
|
+
EM.stop
|
58
|
+
end
|
59
|
+
end
|
39
60
|
end
|
40
61
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
data[key] = values if key
|
62
|
+
describe "multiple master" do
|
63
|
+
before do
|
64
|
+
@pid1 = fork do
|
65
|
+
master = MapReduce::Master.new socket: "ipc:///dev/shm/sock1.sock"
|
66
|
+
master.run
|
67
|
+
end
|
68
|
+
@pid2 = fork do
|
69
|
+
master = MapReduce::Master.new socket: "ipc:///dev/shm/sock2.sock"
|
70
|
+
master.run
|
51
71
|
end
|
52
|
-
|
53
|
-
|
54
|
-
|
72
|
+
end
|
73
|
+
|
74
|
+
after do
|
75
|
+
Process.kill "TERM", @pid1
|
76
|
+
Process.kill "TERM", @pid2
|
77
|
+
end
|
78
|
+
|
79
|
+
it "should map and reduce some data in SYNC mode" do
|
80
|
+
EM.synchrony do
|
81
|
+
data = {}
|
82
|
+
worker = MapReduce::Worker.new type: :sync, masters: ["ipc:///dev/shm/sock1.sock", "ipc:///dev/shm/sock2.sock"]
|
83
|
+
worker.map("Petr", ["Radiohead", "Muse", "R.E.M."] * ',')
|
84
|
+
worker.map("Alex", ["Madonna", "Lady Gaga"] * ',')
|
85
|
+
worker.map("Petr", ["Radiohead", "The Beatles", "Aquarium"] * ',')
|
86
|
+
worker.map("Michael", ["Blur"] * ',')
|
87
|
+
worker.map("Gosha", ["DDT", "Splin"] * ',')
|
88
|
+
worker.map("Obama", ["Adele", "Rolling Stones"] * ',')
|
89
|
+
worker.map_finished
|
90
|
+
worker.reduce do |key, values|
|
91
|
+
data[key] = values if key
|
92
|
+
end
|
93
|
+
data.size.must_equal 5
|
94
|
+
data["Petr"].must_equal [["Radiohead", "Muse", "R.E.M."] * ',', ["Radiohead", "The Beatles", "Aquarium"] * ',']
|
95
|
+
data["Alex"].must_equal [["Madonna", "Lady Gaga"] * ',']
|
55
96
|
|
56
|
-
|
97
|
+
EM.stop
|
98
|
+
end
|
57
99
|
end
|
58
100
|
end
|
59
101
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: map_reduce
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.1.
|
4
|
+
version: 0.0.1.alpha2
|
5
5
|
prerelease: 6
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -110,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
110
110
|
requirements:
|
111
111
|
- - ! '>='
|
112
112
|
- !ruby/object:Gem::Version
|
113
|
-
hash:
|
113
|
+
hash: 2663364217523840830
|
114
114
|
version: '0'
|
115
115
|
segments:
|
116
116
|
- 0
|