tem_mr_search 0.3.2 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/CHANGELOG CHANGED
@@ -1,3 +1,5 @@
1
+ v0.3.3. Map-Reduce scheduler tweak.
2
+
1
3
  v0.3.2. Map-Reduce timing statistics available in the results.
2
4
 
3
5
  v0.3.1. Optimized Map-Reduce scheduler. 1.5x speed increase.
data/Manifest CHANGED
@@ -23,7 +23,8 @@ test/test_map_reduce_planner.rb
23
23
  test/test_query_builders.rb
24
24
  testdata/cluster.yml
25
25
  testdata/empty_cluster.yml
26
- testdata/fares.yml
26
+ testdata/fares16.yml
27
+ testdata/fares8.yml
27
28
  testdata/parallel_plan_431.yml
28
29
  testdata/parallel_plan_740.yml
29
30
  testdata/serial_plan_410.yml
@@ -109,8 +109,8 @@ class MapReducePlanner
109
109
  def next_actions!
110
110
  actions = migrate_actions :mapper
111
111
  actions += migrate_actions :reducer
112
- actions += map_actions
113
112
  actions += reduce_actions
113
+ actions += map_actions
114
114
  actions += finalize_actions
115
115
  actions
116
116
  end
@@ -2,7 +2,7 @@
2
2
 
3
3
  Gem::Specification.new do |s|
4
4
  s.name = %q{tem_mr_search}
5
- s.version = "0.3.2"
5
+ s.version = "0.3.3"
6
6
 
7
7
  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
8
8
  s.authors = ["Victor Costan"]
@@ -12,7 +12,7 @@ Gem::Specification.new do |s|
12
12
  s.email = %q{victor@costan.us}
13
13
  s.executables = ["tem_mr_search_server"]
14
14
  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb"]
15
- s.files = ["CHANGELOG", "LICENSE", "Manifest", "README", "Rakefile", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml", "tem_mr_search.gemspec"]
15
+ s.files = ["CHANGELOG", "LICENSE", "Manifest", "README", "Rakefile", "bin/tem_mr_search_server", "lib/tem_mr_search.rb", "lib/tem_mr_search/client.rb", "lib/tem_mr_search/client_query.rb", "lib/tem_mr_search/db.rb", "lib/tem_mr_search/map_reduce_executor.rb", "lib/tem_mr_search/map_reduce_job.rb", "lib/tem_mr_search/map_reduce_planner.rb", "lib/tem_mr_search/query_builder.rb", "lib/tem_mr_search/server.rb", "lib/tem_mr_search/web_client_query_builder.rb", "test/mr_test_case.rb", "test/test_client_server.rb", "test/test_db.rb", "test/test_map_reduce_executor.rb", "test/test_map_reduce_job.rb", "test/test_map_reduce_planner.rb", "test/test_query_builders.rb", "testdata/cluster.yml", "testdata/empty_cluster.yml", "testdata/fares16.yml", "testdata/fares8.yml", "testdata/parallel_plan_431.yml", "testdata/parallel_plan_740.yml", "testdata/serial_plan_410.yml", "testdata/serial_plan_431.yml", "testdata/serial_plan_740.yml", "tem_mr_search.gemspec"]
16
16
  s.homepage = %q{http://tem.rubyforge.org}
17
17
  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Tem_mr_search", "--main", "README"]
18
18
  s.require_paths = ["lib"]
data/test/mr_test_case.rb CHANGED
@@ -9,9 +9,9 @@ class MrTestCase < Test::Unit::TestCase
9
9
 
10
10
  Thread.abort_on_exception = true
11
11
 
12
- @db_path = File.join File.dirname(__FILE__), "..", "testdata", "fares.yml"
12
+ @db_path = File.join File.dirname(__FILE__), "..", "testdata", "fares8.yml"
13
13
  @cluster_file = File.join File.dirname(__FILE__), "..", "testdata",
14
- "cluster.yml"
14
+ "cluster.yml"
15
15
  @empty_cluster_file = File.join File.dirname(__FILE__), "..", "testdata",
16
16
  "empty_cluster.yml"
17
17
  @db = Db.new @db_path
@@ -1,7 +1,6 @@
1
1
  require 'test/mr_test_case'
2
2
  require 'flexmock/test_unit'
3
3
 
4
-
5
4
  class ClientServerTest < MrTestCase
6
5
  Client = Tem::Mr::Search::Client
7
6
  Server = Tem::Mr::Search::Server
@@ -0,0 +1,113 @@
1
+ ---
2
+ - from: BOS
3
+ to: TPE
4
+ price: 2500
5
+ start_time: 900
6
+ end_time: 2100
7
+ layovers: 2
8
+ flight: 15
9
+ - from: BOS
10
+ to: TPE
11
+ price: 2500
12
+ start_time: 1100
13
+ end_time: 2300
14
+ layovers: 2
15
+ flight: 16
16
+ - from: BOS
17
+ to: TPE
18
+ price: 3000
19
+ start_time: 900
20
+ end_time: 1900
21
+ layovers: 1
22
+ flight: 17
23
+ - from: BOS
24
+ to: TPE
25
+ price: 3000
26
+ start_time: 1100
27
+ end_time: 2100
28
+ layovers: 1
29
+ flight: 18
30
+ - from: BOS
31
+ to: TPE
32
+ price: 4000
33
+ start_time: 900
34
+ end_time: 1700
35
+ layovers: 0
36
+ flight: 19
37
+ - from: BOS
38
+ to: TPE
39
+ price: 4000
40
+ start_time: 1100
41
+ end_time: 1900
42
+ layovers: 0
43
+ flight: 20
44
+ - from: BOS
45
+ to: TPE
46
+ price: 2400
47
+ start_time: 900
48
+ end_time: 2100
49
+ layovers: 2
50
+ flight: 21
51
+ - from: BOS
52
+ to: TPE
53
+ price: 2400
54
+ start_time: 1100
55
+ end_time: 2300
56
+ layovers: 2
57
+ flight: 22
58
+ - from: BOS
59
+ to: TPE
60
+ price: 2500
61
+ start_time: 900
62
+ end_time: 2100
63
+ layovers: 2
64
+ flight: 15
65
+ - from: BOS
66
+ to: TPE
67
+ price: 2500
68
+ start_time: 1100
69
+ end_time: 2300
70
+ layovers: 2
71
+ flight: 16
72
+ - from: BOS
73
+ to: TPE
74
+ price: 3000
75
+ start_time: 900
76
+ end_time: 1900
77
+ layovers: 1
78
+ flight: 17
79
+ - from: BOS
80
+ to: TPE
81
+ price: 3000
82
+ start_time: 1100
83
+ end_time: 2100
84
+ layovers: 1
85
+ flight: 18
86
+ - from: BOS
87
+ to: TPE
88
+ price: 4000
89
+ start_time: 900
90
+ end_time: 1700
91
+ layovers: 0
92
+ flight: 19
93
+ - from: BOS
94
+ to: TPE
95
+ price: 4000
96
+ start_time: 1100
97
+ end_time: 1900
98
+ layovers: 0
99
+ flight: 20
100
+ - from: BOS
101
+ to: TPE
102
+ price: 2400
103
+ start_time: 900
104
+ end_time: 2100
105
+ layovers: 2
106
+ flight: 21
107
+ - from: BOS
108
+ to: TPE
109
+ price: 2400
110
+ start_time: 1100
111
+ end_time: 2300
112
+ layovers: 2
113
+ flight: 22
File without changes
@@ -1,52 +1,52 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 1
2
+ - - :with: 1
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 0
6
- - :action: :migrate
7
- :with: 2
6
+ - :with: 2
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 0
10
- - - :action: :migrate
11
- :with: 1
10
+ - - :with: 1
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :migrate
15
- :with: 0
14
+ - :with: 0
15
+ :action: :migrate
16
16
  :secpack: :reducer
17
17
  :to: 1
18
- - - :action: :map
19
- :item: 0
18
+ - - :item: 0
20
19
  :with: 1
20
+ :action: :map
21
21
  :output_id: 0
22
- - :action: :map
23
- :item: 1
22
+ - :item: 1
24
23
  :with: 0
24
+ :action: :map
25
25
  :output_id: 1
26
- - :action: :map
27
- :item: 2
26
+ - :item: 2
28
27
  :with: 2
28
+ :action: :map
29
29
  :output_id: 2
30
- - - :action: :map
31
- :item: 3
32
- :with: 1
33
- :output_id: 3
34
- - :action: :reduce
30
+ - - :output2_id: 1
35
31
  :with: 2
36
- :output_id: 4
32
+ :action: :reduce
33
+ :output_id: 3
37
34
  :output1_id: 0
38
- :output2_id: 1
39
- - - :action: :reduce
35
+ - :item: 3
36
+ :with: 1
37
+ :action: :map
38
+ :output_id: 4
39
+ - - :output2_id: 3
40
40
  :with: 2
41
+ :action: :reduce
41
42
  :output_id: 5
42
43
  :output1_id: 2
43
- :output2_id: 3
44
- - - :action: :reduce
44
+ - - :output2_id: 5
45
45
  :with: 2
46
+ :action: :reduce
46
47
  :output_id: 6
47
48
  :output1_id: 4
48
- :output2_id: 5
49
- - - :action: :finalize
50
- :with: 0
49
+ - - :with: 0
50
+ :action: :finalize
51
51
  :output_id: 6
52
52
  :final_id: 7
@@ -1,87 +1,87 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 0
2
+ - - :with: 0
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 1
6
- - :action: :migrate
7
- :with: 3
6
+ - :with: 3
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 2
10
- - - :action: :migrate
11
- :with: 0
10
+ - - :with: 0
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :migrate
15
- :with: 1
14
+ - :with: 1
15
+ :action: :migrate
16
16
  :secpack: :mapper
17
17
  :to: 3
18
- - :action: :migrate
19
- :with: 2
18
+ - :with: 2
19
+ :action: :migrate
20
20
  :secpack: :reducer
21
21
  :to: 1
22
- - :action: :migrate
23
- :with: 3
22
+ - :with: 3
23
+ :action: :migrate
24
24
  :secpack: :reducer
25
25
  :to: 0
26
- - - :action: :map
27
- :item: 0
26
+ - - :item: 0
28
27
  :with: 0
28
+ :action: :map
29
29
  :output_id: 0
30
- - :action: :map
31
- :item: 1
30
+ - :item: 1
32
31
  :with: 1
32
+ :action: :map
33
33
  :output_id: 1
34
- - :action: :map
35
- :item: 2
34
+ - :item: 2
36
35
  :with: 2
36
+ :action: :map
37
37
  :output_id: 2
38
- - :action: :map
39
- :item: 3
38
+ - :item: 3
40
39
  :with: 3
40
+ :action: :map
41
41
  :output_id: 3
42
- - - :action: :map
43
- :item: 4
44
- :with: 0
42
+ - - :output2_id: 1
43
+ :with: 3
44
+ :action: :reduce
45
45
  :output_id: 4
46
- - :action: :map
47
- :item: 5
48
- :with: 1
49
- :output_id: 5
50
- - :action: :map
51
- :item: 6
46
+ :output1_id: 0
47
+ - :output2_id: 3
52
48
  :with: 2
49
+ :action: :reduce
50
+ :output_id: 5
51
+ :output1_id: 2
52
+ - :item: 4
53
+ :with: 0
54
+ :action: :map
53
55
  :output_id: 6
54
- - :action: :reduce
55
- :with: 3
56
+ - :item: 5
57
+ :with: 1
58
+ :action: :map
56
59
  :output_id: 7
57
- :output1_id: 0
58
- :output2_id: 1
59
- - - :action: :reduce
60
+ - - :output2_id: 5
60
61
  :with: 3
62
+ :action: :reduce
61
63
  :output_id: 8
62
- :output1_id: 2
63
- :output2_id: 3
64
- - :action: :reduce
64
+ :output1_id: 4
65
+ - :output2_id: 7
65
66
  :with: 2
67
+ :action: :reduce
66
68
  :output_id: 9
67
- :output1_id: 4
68
- :output2_id: 5
69
- - :action: :reduce
70
- :with: 1
71
- :output_id: 10
72
69
  :output1_id: 6
73
- :output2_id: 7
74
- - - :action: :reduce
70
+ - :item: 6
71
+ :with: 0
72
+ :action: :map
73
+ :output_id: 10
74
+ - - :output2_id: 9
75
75
  :with: 3
76
+ :action: :reduce
76
77
  :output_id: 11
77
78
  :output1_id: 8
78
- :output2_id: 9
79
- - - :action: :reduce
79
+ - - :output2_id: 11
80
80
  :with: 3
81
+ :action: :reduce
81
82
  :output_id: 12
82
83
  :output1_id: 10
83
- :output2_id: 11
84
- - - :action: :finalize
85
- :with: 1
84
+ - - :with: 1
85
+ :action: :finalize
86
86
  :output_id: 12
87
87
  :final_id: 13
@@ -1,36 +1,36 @@
1
1
  ---
2
- - - :action: :map
3
- :item: 0
2
+ - - :item: 0
4
3
  :with: 0
4
+ :action: :map
5
5
  :output_id: 0
6
- - - :action: :map
7
- :item: 1
6
+ - - :item: 1
8
7
  :with: 0
8
+ :action: :map
9
9
  :output_id: 1
10
- - - :action: :map
11
- :item: 2
10
+ - - :output2_id: 1
12
11
  :with: 0
12
+ :action: :reduce
13
13
  :output_id: 2
14
- - - :action: :map
15
- :item: 3
14
+ :output1_id: 0
15
+ - - :item: 2
16
16
  :with: 0
17
+ :action: :map
17
18
  :output_id: 3
18
- - - :action: :reduce
19
+ - - :output2_id: 3
19
20
  :with: 0
21
+ :action: :reduce
20
22
  :output_id: 4
21
- :output1_id: 0
22
- :output2_id: 1
23
- - - :action: :reduce
23
+ :output1_id: 2
24
+ - - :item: 3
24
25
  :with: 0
26
+ :action: :map
25
27
  :output_id: 5
26
- :output1_id: 2
27
- :output2_id: 3
28
- - - :action: :reduce
28
+ - - :output2_id: 5
29
29
  :with: 0
30
+ :action: :reduce
30
31
  :output_id: 6
31
32
  :output1_id: 4
32
- :output2_id: 5
33
- - - :action: :finalize
34
- :with: 0
33
+ - - :with: 0
34
+ :action: :finalize
35
35
  :output_id: 6
36
36
  :final_id: 7
@@ -1,56 +1,56 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 0
2
+ - - :with: 0
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 1
6
- - :action: :migrate
7
- :with: 2
6
+ - :with: 2
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 1
10
- - - :action: :migrate
11
- :with: 0
10
+ - - :with: 0
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :map
15
- :item: 0
14
+ - :item: 0
16
15
  :with: 1
16
+ :action: :map
17
17
  :output_id: 0
18
- - - :action: :migrate
19
- :with: 2
18
+ - - :with: 2
19
+ :action: :migrate
20
20
  :secpack: :reducer
21
21
  :to: 0
22
- - - :action: :map
23
- :item: 1
22
+ - - :item: 1
24
23
  :with: 0
24
+ :action: :map
25
25
  :output_id: 1
26
- - - :action: :map
27
- :item: 2
26
+ - - :item: 2
28
27
  :with: 1
28
+ :action: :map
29
29
  :output_id: 2
30
- - - :action: :map
31
- :item: 3
30
+ - - :item: 3
32
31
  :with: 2
32
+ :action: :map
33
33
  :output_id: 3
34
- - - :action: :reduce
34
+ - - :output2_id: 1
35
35
  :with: 0
36
+ :action: :reduce
36
37
  :output_id: 4
37
38
  :output1_id: 0
38
- :output2_id: 1
39
39
  - []
40
40
 
41
- - - :action: :reduce
41
+ - - :output2_id: 3
42
42
  :with: 2
43
+ :action: :reduce
43
44
  :output_id: 5
44
45
  :output1_id: 2
45
- :output2_id: 3
46
46
  - []
47
47
 
48
- - - :action: :reduce
48
+ - - :output2_id: 5
49
49
  :with: 2
50
+ :action: :reduce
50
51
  :output_id: 6
51
52
  :output1_id: 4
52
- :output2_id: 5
53
- - - :action: :finalize
54
- :with: 1
53
+ - - :with: 1
54
+ :action: :finalize
55
55
  :output_id: 6
56
56
  :final_id: 7
@@ -1,93 +1,93 @@
1
1
  ---
2
- - - :action: :migrate
3
- :with: 0
2
+ - - :with: 0
3
+ :action: :migrate
4
4
  :secpack: :mapper
5
5
  :to: 1
6
- - :action: :migrate
7
- :with: 3
6
+ - :with: 3
7
+ :action: :migrate
8
8
  :secpack: :reducer
9
9
  :to: 2
10
- - - :action: :migrate
11
- :with: 0
10
+ - - :with: 0
11
+ :action: :migrate
12
12
  :secpack: :mapper
13
13
  :to: 2
14
- - :action: :migrate
15
- :with: 1
14
+ - :with: 1
15
+ :action: :migrate
16
16
  :secpack: :mapper
17
17
  :to: 3
18
- - - :action: :migrate
19
- :with: 2
18
+ - - :with: 2
19
+ :action: :migrate
20
20
  :secpack: :reducer
21
21
  :to: 1
22
- - :action: :migrate
23
- :with: 3
22
+ - :with: 3
23
+ :action: :migrate
24
24
  :secpack: :reducer
25
25
  :to: 0
26
- - - :action: :map
27
- :item: 0
26
+ - - :item: 0
28
27
  :with: 0
28
+ :action: :map
29
29
  :output_id: 0
30
- - - :action: :map
31
- :item: 1
30
+ - - :item: 1
32
31
  :with: 1
32
+ :action: :map
33
33
  :output_id: 1
34
- - - :action: :map
35
- :item: 2
34
+ - - :item: 2
36
35
  :with: 2
36
+ :action: :map
37
37
  :output_id: 2
38
- - - :action: :map
39
- :item: 3
38
+ - - :item: 3
40
39
  :with: 3
40
+ :action: :map
41
41
  :output_id: 3
42
- - - :action: :map
43
- :item: 4
42
+ - - :item: 4
44
43
  :with: 0
44
+ :action: :map
45
45
  :output_id: 4
46
- - - :action: :map
47
- :item: 5
46
+ - - :output2_id: 1
48
47
  :with: 1
48
+ :action: :reduce
49
49
  :output_id: 5
50
- - - :action: :map
51
- :item: 6
50
+ :output1_id: 0
51
+ - - :item: 5
52
52
  :with: 2
53
+ :action: :map
53
54
  :output_id: 6
54
- - - :action: :reduce
55
+ - - :output2_id: 3
55
56
  :with: 3
57
+ :action: :reduce
56
58
  :output_id: 7
57
- :output1_id: 0
58
- :output2_id: 1
59
- - - :action: :reduce
59
+ :output1_id: 2
60
+ - - :item: 6
60
61
  :with: 0
62
+ :action: :map
61
63
  :output_id: 8
62
- :output1_id: 2
63
- :output2_id: 3
64
- - - :action: :reduce
64
+ - - :output2_id: 5
65
65
  :with: 1
66
+ :action: :reduce
66
67
  :output_id: 9
67
68
  :output1_id: 4
68
- :output2_id: 5
69
69
  - []
70
70
 
71
- - - :action: :reduce
71
+ - - :output2_id: 7
72
72
  :with: 3
73
+ :action: :reduce
73
74
  :output_id: 10
74
75
  :output1_id: 6
75
- :output2_id: 7
76
76
  - []
77
77
 
78
- - - :action: :reduce
78
+ - - :output2_id: 9
79
79
  :with: 2
80
+ :action: :reduce
80
81
  :output_id: 11
81
82
  :output1_id: 8
82
- :output2_id: 9
83
83
  - []
84
84
 
85
- - - :action: :reduce
85
+ - - :output2_id: 11
86
86
  :with: 3
87
+ :action: :reduce
87
88
  :output_id: 12
88
89
  :output1_id: 10
89
- :output2_id: 11
90
- - - :action: :finalize
91
- :with: 0
90
+ - - :with: 0
91
+ :action: :finalize
92
92
  :output_id: 12
93
93
  :final_id: 13
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tem_mr_search
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.3.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Victor Costan
@@ -99,7 +99,8 @@ files:
99
99
  - test/test_query_builders.rb
100
100
  - testdata/cluster.yml
101
101
  - testdata/empty_cluster.yml
102
- - testdata/fares.yml
102
+ - testdata/fares16.yml
103
+ - testdata/fares8.yml
103
104
  - testdata/parallel_plan_431.yml
104
105
  - testdata/parallel_plan_740.yml
105
106
  - testdata/serial_plan_410.yml