tem_mr_search 0.3.2 → 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
data/CHANGELOG CHANGED
@@ -1,3 +1,5 @@
1
+ v0.3.3. Map-Reduce scheduler tweak.
2
+
1
3
  v0.3.2. Map-Reduce timing statistics available in the results.
2
4
 
3
5
  v0.3.1. Optimized Map-Reduce scheduler. 1.5x speed increase.
data/Manifest CHANGED
@@ -23,7 +23,8 @@ test/test_map_reduce_planner.rb
23
23
  test/test_query_builders.rb
24
24
  testdata/cluster.yml
25
25
  testdata/empty_cluster.yml
26
- testdata/fares.yml
26
+ testdata/fares16.yml
27
+ testdata/fares8.yml
27
28
  testdata/parallel_plan_431.yml
28
29
  testdata/parallel_plan_740.yml
29
30
  testdata/serial_plan_410.yml
@@ -109,8 +109,8 @@ class MapReducePlanner
109
109
  def next_actions!
110
110
  actions = migrate_actions :mapper
111
111
  actions += migrate_actions :reducer
112
- actions += map_actions
113
112
  actions += reduce_actions
113
+ actions += map_actions
114
114
  actions += finalize_actions
115
115
  actions
116
116
  end
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = %q{tem_mr_search}
5
- s.version = "0.3.2"
5
+ s.version = "0.3.3"
6
6
 
7
7
  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
8
8
  s.authors = ["Victor Costan"]
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
12
12
  s.email = %q{victor@costan.us}
13
13
  s.executables = ["tem_mr_search_server"]
14
14
  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb"]
15
- s.files = ["CHANGELOG", "LICENSE", "Manifest", "README", "Rakefile", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml", "tem_mr_search.gemspec"]
15
+ s.files = ["CHANGELOG", "LICENSE", "Manifest", "README", "Rakefile", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares16.yml", "testdata/fares8.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml", "tem_mr_search.gemspec"]
16
16
  s.homepage = %q{http://tem.rubyforge.org}
17
17
  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Tem_mr_search", "--main", "README"]
18
18
  s.require_paths = ["lib"]
data/test/mr_test_case.rb CHANGED
@@ -9,9 +9,9 @@ class MrTestCase < Test::Unit::TestCase
9
9
 
10
10
  Thread.abort_on_exception = true
11
11
 
12
- @db_path = File.join File.dirname(__FILE__), "..", "testdata", "fares.yml"
12
+ @db_path = File.join File.dirname(__FILE__), "..", "testdata", "fares8.yml"
13
13
  @cluster_file = File.join File.dirname(__FILE__), "..", "testdata",
14
- "cluster.yml"
14
+ "cluster.yml"
15
15
  @empty_cluster_file = File.join File.dirname(__FILE__), "..", "testdata",
16
16
  "empty_cluster.yml"
17
17
  @db = Db.new @db_path
@@ -1,7 +1,6 @@
1
1
  require 'test/mr_test_case'
2
2
  require 'flexmock/test_unit'
3
3
 
4
-
5
4
  class ClientServerTest < MrTestCase
6
5
  Client = Tem::Mr::Search::Client
7
6
  Server = Tem::Mr::Search::Server
@@ -0,0 +1,113 @@
1
+ ---
2
+ - from: BOS
3
+ to: TPE
4
+ price: 2500
5
+ start_time: 900
6
+ end_time: 2100
7
+ layovers: 2
8
+ flight: 15
9
+ - from: BOS
10
+ to: TPE
11
+ price: 2500
12
+ start_time: 1100
13
+ end_time: 2300
14
+ layovers: 2
15
+ flight: 16
16
+ - from: BOS
17
+ to: TPE
18
+ price: 3000
19
+ start_time: 900
20
+ end_time: 1900
21
+ layovers: 1
22
+ flight: 17
23
+ - from: BOS
24
+ to: TPE
25
+ price: 3000
26
+ start_time: 1100
27
+ end_time: 2100
28
+ layovers: 1
29
+ flight: 18
30
+ - from: BOS
31
+ to: TPE
32
+ price: 4000
33
+ start_time: 900
34
+ end_time: 1700
35
+ layovers: 0
36
+ flight: 19
37
+ - from: BOS
38
+ to: TPE
39
+ price: 4000
40
+ start_time: 1100
41
+ end_time: 1900
42
+ layovers: 0
43
+ flight: 20
44
+ - from: BOS
45
+ to: TPE
46
+ price: 2400
47
+ start_time: 900
48
+ end_time: 2100
49
+ layovers: 2
50
+ flight: 21
51
+ - from: BOS
52
+ to: TPE
53
+ price: 2400
54
+ start_time: 1100
55
+ end_time: 2300
56
+ layovers: 2
57
+ flight: 22
58
+ - from: BOS
59
+ to: TPE
60
+ price: 2500
61
+ start_time: 900
62
+ end_time: 2100
63
+ layovers: 2
64
+ flight: 15
65
+ - from: BOS
66
+ to: TPE
67
+ price: 2500
68
+ start_time: 1100
69
+ end_time: 2300
70
+ layovers: 2
71
+ flight: 16
72
+ - from: BOS
73
+ to: TPE
74
+ price: 3000
75
+ start_time: 900
76
+ end_time: 1900
77
+ layovers: 1
78
+ flight: 17
79
+ - from: BOS
80
+ to: TPE
81
+ price: 3000
82
+ start_time: 1100
83
+ end_time: 2100
84
+ layovers: 1
85
+ flight: 18
86
+ - from: BOS
87
+ to: TPE
88
+ price: 4000
89
+ start_time: 900
90
+ end_time: 1700
91
+ layovers: 0
92
+ flight: 19
93
+ - from: BOS
94
+ to: TPE
95
+ price: 4000
96
+ start_time: 1100
97
+ end_time: 1900
98
+ layovers: 0
99
+ flight: 20
100
+ - from: BOS
101
+ to: TPE
102
+ price: 2400
103
+ start_time: 900
104
+ end_time: 2100
105
+ layovers: 2
106
+ flight: 21
107
+ - from: BOS
108
+ to: TPE
109
+ price: 2400
110
+ start_time: 1100
111
+ end_time: 2300
112
+ layovers: 2
113
+ flight: 22
File without changes
@@ -1,52 +1,52 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 1
2
+ - - :with: 1
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 0
6
- - :action: :migrate
7
- :with: 2
6
+ - :with: 2
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 0
10
- - - :action: :migrate
11
- :with: 1
10
+ - - :with: 1
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :migrate
15
- :with: 0
14
+ - :with: 0
15
+ :action: :migrate
16
16
  :secpack: :reducer
17
17
  :to: 1
18
- - - :action: :map
19
- :item: 0
18
+ - - :item: 0
20
19
  :with: 1
20
+ :action: :map
21
21
  :output_id: 0
22
- - :action: :map
23
- :item: 1
22
+ - :item: 1
24
23
  :with: 0
24
+ :action: :map
25
25
  :output_id: 1
26
- - :action: :map
27
- :item: 2
26
+ - :item: 2
28
27
  :with: 2
28
+ :action: :map
29
29
  :output_id: 2
30
- - - :action: :map
31
- :item: 3
32
- :with: 1
33
- :output_id: 3
34
- - :action: :reduce
30
+ - - :output2_id: 1
35
31
  :with: 2
36
- :output_id: 4
32
+ :action: :reduce
33
+ :output_id: 3
37
34
  :output1_id: 0
38
- :output2_id: 1
39
- - - :action: :reduce
35
+ - :item: 3
36
+ :with: 1
37
+ :action: :map
38
+ :output_id: 4
39
+ - - :output2_id: 3
40
40
  :with: 2
41
+ :action: :reduce
41
42
  :output_id: 5
42
43
  :output1_id: 2
43
- :output2_id: 3
44
- - - :action: :reduce
44
+ - - :output2_id: 5
45
45
  :with: 2
46
+ :action: :reduce
46
47
  :output_id: 6
47
48
  :output1_id: 4
48
- :output2_id: 5
49
- - - :action: :finalize
50
- :with: 0
49
+ - - :with: 0
50
+ :action: :finalize
51
51
  :output_id: 6
52
52
  :final_id: 7
@@ -1,87 +1,87 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 0
2
+ - - :with: 0
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 1
6
- - :action: :migrate
7
- :with: 3
6
+ - :with: 3
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 2
10
- - - :action: :migrate
11
- :with: 0
10
+ - - :with: 0
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :migrate
15
- :with: 1
14
+ - :with: 1
15
+ :action: :migrate
16
16
  :secpack: :mapper
17
17
  :to: 3
18
- - :action: :migrate
19
- :with: 2
18
+ - :with: 2
19
+ :action: :migrate
20
20
  :secpack: :reducer
21
21
  :to: 1
22
- - :action: :migrate
23
- :with: 3
22
+ - :with: 3
23
+ :action: :migrate
24
24
  :secpack: :reducer
25
25
  :to: 0
26
- - - :action: :map
27
- :item: 0
26
+ - - :item: 0
28
27
  :with: 0
28
+ :action: :map
29
29
  :output_id: 0
30
- - :action: :map
31
- :item: 1
30
+ - :item: 1
32
31
  :with: 1
32
+ :action: :map
33
33
  :output_id: 1
34
- - :action: :map
35
- :item: 2
34
+ - :item: 2
36
35
  :with: 2
36
+ :action: :map
37
37
  :output_id: 2
38
- - :action: :map
39
- :item: 3
38
+ - :item: 3
40
39
  :with: 3
40
+ :action: :map
41
41
  :output_id: 3
42
- - - :action: :map
43
- :item: 4
44
- :with: 0
42
+ - - :output2_id: 1
43
+ :with: 3
44
+ :action: :reduce
45
45
  :output_id: 4
46
- - :action: :map
47
- :item: 5
48
- :with: 1
49
- :output_id: 5
50
- - :action: :map
51
- :item: 6
46
+ :output1_id: 0
47
+ - :output2_id: 3
52
48
  :with: 2
49
+ :action: :reduce
50
+ :output_id: 5
51
+ :output1_id: 2
52
+ - :item: 4
53
+ :with: 0
54
+ :action: :map
53
55
  :output_id: 6
54
- - :action: :reduce
55
- :with: 3
56
+ - :item: 5
57
+ :with: 1
58
+ :action: :map
56
59
  :output_id: 7
57
- :output1_id: 0
58
- :output2_id: 1
59
- - - :action: :reduce
60
+ - - :output2_id: 5
60
61
  :with: 3
62
+ :action: :reduce
61
63
  :output_id: 8
62
- :output1_id: 2
63
- :output2_id: 3
64
- - :action: :reduce
64
+ :output1_id: 4
65
+ - :output2_id: 7
65
66
  :with: 2
67
+ :action: :reduce
66
68
  :output_id: 9
67
- :output1_id: 4
68
- :output2_id: 5
69
- - :action: :reduce
70
- :with: 1
71
- :output_id: 10
72
69
  :output1_id: 6
73
- :output2_id: 7
74
- - - :action: :reduce
70
+ - :item: 6
71
+ :with: 0
72
+ :action: :map
73
+ :output_id: 10
74
+ - - :output2_id: 9
75
75
  :with: 3
76
+ :action: :reduce
76
77
  :output_id: 11
77
78
  :output1_id: 8
78
- :output2_id: 9
79
- - - :action: :reduce
79
+ - - :output2_id: 11
80
80
  :with: 3
81
+ :action: :reduce
81
82
  :output_id: 12
82
83
  :output1_id: 10
83
- :output2_id: 11
84
- - - :action: :finalize
85
- :with: 1
84
+ - - :with: 1
85
+ :action: :finalize
86
86
  :output_id: 12
87
87
  :final_id: 13
@@ -1,36 +1,36 @@
1
1
  ---
2
- - - :action: :map
3
- :item: 0
2
+ - - :item: 0
4
3
  :with: 0
4
+ :action: :map
5
5
  :output_id: 0
6
- - - :action: :map
7
- :item: 1
6
+ - - :item: 1
8
7
  :with: 0
8
+ :action: :map
9
9
  :output_id: 1
10
- - - :action: :map
11
- :item: 2
10
+ - - :output2_id: 1
12
11
  :with: 0
12
+ :action: :reduce
13
13
  :output_id: 2
14
- - - :action: :map
15
- :item: 3
14
+ :output1_id: 0
15
+ - - :item: 2
16
16
  :with: 0
17
+ :action: :map
17
18
  :output_id: 3
18
- - - :action: :reduce
19
+ - - :output2_id: 3
19
20
  :with: 0
21
+ :action: :reduce
20
22
  :output_id: 4
21
- :output1_id: 0
22
- :output2_id: 1
23
- - - :action: :reduce
23
+ :output1_id: 2
24
+ - - :item: 3
24
25
  :with: 0
26
+ :action: :map
25
27
  :output_id: 5
26
- :output1_id: 2
27
- :output2_id: 3
28
- - - :action: :reduce
28
+ - - :output2_id: 5
29
29
  :with: 0
30
+ :action: :reduce
30
31
  :output_id: 6
31
32
  :output1_id: 4
32
- :output2_id: 5
33
- - - :action: :finalize
34
- :with: 0
33
+ - - :with: 0
34
+ :action: :finalize
35
35
  :output_id: 6
36
36
  :final_id: 7
@@ -1,56 +1,56 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 0
2
+ - - :with: 0
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 1
6
- - :action: :migrate
7
- :with: 2
6
+ - :with: 2
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 1
10
- - - :action: :migrate
11
- :with: 0
10
+ - - :with: 0
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :map
15
- :item: 0
14
+ - :item: 0
16
15
  :with: 1
16
+ :action: :map
17
17
  :output_id: 0
18
- - - :action: :migrate
19
- :with: 2
18
+ - - :with: 2
19
+ :action: :migrate
20
20
  :secpack: :reducer
21
21
  :to: 0
22
- - - :action: :map
23
- :item: 1
22
+ - - :item: 1
24
23
  :with: 0
24
+ :action: :map
25
25
  :output_id: 1
26
- - - :action: :map
27
- :item: 2
26
+ - - :item: 2
28
27
  :with: 1
28
+ :action: :map
29
29
  :output_id: 2
30
- - - :action: :map
31
- :item: 3
30
+ - - :item: 3
32
31
  :with: 2
32
+ :action: :map
33
33
  :output_id: 3
34
- - - :action: :reduce
34
+ - - :output2_id: 1
35
35
  :with: 0
36
+ :action: :reduce
36
37
  :output_id: 4
37
38
  :output1_id: 0
38
- :output2_id: 1
39
39
  - []
40
40
 
41
- - - :action: :reduce
41
+ - - :output2_id: 3
42
42
  :with: 2
43
+ :action: :reduce
43
44
  :output_id: 5
44
45
  :output1_id: 2
45
- :output2_id: 3
46
46
  - []
47
47
 
48
- - - :action: :reduce
48
+ - - :output2_id: 5
49
49
  :with: 2
50
+ :action: :reduce
50
51
  :output_id: 6
51
52
  :output1_id: 4
52
- :output2_id: 5
53
- - - :action: :finalize
54
- :with: 1
53
+ - - :with: 1
54
+ :action: :finalize
55
55
  :output_id: 6
56
56
  :final_id: 7
@@ -1,93 +1,93 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 0
2
+ - - :with: 0
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 1
6
- - :action: :migrate
7
- :with: 3
6
+ - :with: 3
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 2
10
- - - :action: :migrate
11
- :with: 0
10
+ - - :with: 0
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :migrate
15
- :with: 1
14
+ - :with: 1
15
+ :action: :migrate
16
16
  :secpack: :mapper
17
17
  :to: 3
18
- - - :action: :migrate
19
- :with: 2
18
+ - - :with: 2
19
+ :action: :migrate
20
20
  :secpack: :reducer
21
21
  :to: 1
22
- - :action: :migrate
23
- :with: 3
22
+ - :with: 3
23
+ :action: :migrate
24
24
  :secpack: :reducer
25
25
  :to: 0
26
- - - :action: :map
27
- :item: 0
26
+ - - :item: 0
28
27
  :with: 0
28
+ :action: :map
29
29
  :output_id: 0
30
- - - :action: :map
31
- :item: 1
30
+ - - :item: 1
32
31
  :with: 1
32
+ :action: :map
33
33
  :output_id: 1
34
- - - :action: :map
35
- :item: 2
34
+ - - :item: 2
36
35
  :with: 2
36
+ :action: :map
37
37
  :output_id: 2
38
- - - :action: :map
39
- :item: 3
38
+ - - :item: 3
40
39
  :with: 3
40
+ :action: :map
41
41
  :output_id: 3
42
- - - :action: :map
43
- :item: 4
42
+ - - :item: 4
44
43
  :with: 0
44
+ :action: :map
45
45
  :output_id: 4
46
- - - :action: :map
47
- :item: 5
46
+ - - :output2_id: 1
48
47
  :with: 1
48
+ :action: :reduce
49
49
  :output_id: 5
50
- - - :action: :map
51
- :item: 6
50
+ :output1_id: 0
51
+ - - :item: 5
52
52
  :with: 2
53
+ :action: :map
53
54
  :output_id: 6
54
- - - :action: :reduce
55
+ - - :output2_id: 3
55
56
  :with: 3
57
+ :action: :reduce
56
58
  :output_id: 7
57
- :output1_id: 0
58
- :output2_id: 1
59
- - - :action: :reduce
59
+ :output1_id: 2
60
+ - - :item: 6
60
61
  :with: 0
62
+ :action: :map
61
63
  :output_id: 8
62
- :output1_id: 2
63
- :output2_id: 3
64
- - - :action: :reduce
64
+ - - :output2_id: 5
65
65
  :with: 1
66
+ :action: :reduce
66
67
  :output_id: 9
67
68
  :output1_id: 4
68
- :output2_id: 5
69
69
  - []
70
70
 
71
- - - :action: :reduce
71
+ - - :output2_id: 7
72
72
  :with: 3
73
+ :action: :reduce
73
74
  :output_id: 10
74
75
  :output1_id: 6
75
- :output2_id: 7
76
76
  - []
77
77
 
78
- - - :action: :reduce
78
+ - - :output2_id: 9
79
79
  :with: 2
80
+ :action: :reduce
80
81
  :output_id: 11
81
82
  :output1_id: 8
82
- :output2_id: 9
83
83
  - []
84
84
 
85
- - - :action: :reduce
85
+ - - :output2_id: 11
86
86
  :with: 3
87
+ :action: :reduce
87
88
  :output_id: 12
88
89
  :output1_id: 10
89
- :output2_id: 11
90
- - - :action: :finalize
91
- :with: 0
90
+ - - :with: 0
91
+ :action: :finalize
92
92
  :output_id: 12
93
93
  :final_id: 13
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tem_mr_search
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.3.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Victor Costan
@@ -99,7 +99,8 @@ files:
99
99
  - test/test_query_builders.rb
100
100
  - testdata/cluster.yml
101
101
  - testdata/empty_cluster.yml
102
- - testdata/fares.yml
102
+ - testdata/fares16.yml
103
+ - testdata/fares8.yml
103
104
  - testdata/parallel_plan_431.yml
104
105
  - testdata/parallel_plan_740.yml
105
106
  - testdata/serial_plan_410.yml