cascading.jruby 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- data/HACKING.md +1 -1
- data/History.txt +6 -0
- data/README.md +1 -1
- data/ivy.xml +3 -3
- data/lib/cascading/aggregations.rb +14 -13
- data/lib/cascading/base.rb +27 -9
- data/lib/cascading/cascade.rb +12 -3
- data/lib/cascading/cascading.rb +35 -44
- data/lib/cascading/cascading_exception.rb +5 -5
- data/lib/cascading/flow.rb +23 -32
- data/lib/cascading/mode.rb +78 -0
- data/lib/cascading/operations.rb +10 -4
- data/lib/cascading/scope.rb +8 -2
- data/lib/cascading/sub_assembly.rb +6 -6
- data/lib/cascading/tap.rb +81 -0
- data/lib/cascading.rb +3 -1
- data/samples/branch.rb +2 -2
- data/samples/copy.rb +2 -2
- data/samples/group_by.rb +5 -5
- data/samples/join.rb +2 -2
- data/samples/logwordcount.rb +3 -4
- data/samples/project.rb +2 -2
- data/samples/rename.rb +2 -2
- data/samples/scorenames.rb +2 -2
- data/samples/splitter.rb +2 -2
- data/samples/sub_assembly.rb +2 -2
- data/samples/union.rb +2 -2
- data/spec/cascading_spec.rb +17 -17
- data/spec/spec_util.rb +9 -9
- data/tags +92 -41
- data/tasks/ant.rake +6 -1
- data/test/test_assembly.rb +14 -7
- data/test/test_cascade.rb +55 -0
- data/test/test_cascading.rb +12 -15
- data/test/test_flow.rb +53 -36
- data/test/test_local_execution.rb +7 -7
- data/test/test_operations.rb +61 -0
- metadata +4 -2
data/test/test_flow.rb
CHANGED
@@ -31,40 +31,30 @@ class TC_Flow < Test::Unit::TestCase
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def test_ambiguous_assembly_names
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
ex = assert_raise AmbiguousNodeNameException do
|
35
|
+
a1, a2, x = nil, nil, nil
|
36
|
+
flow = flow 'flow' do
|
37
|
+
source 'a', tap('test/data/data1.txt')
|
37
38
|
|
38
|
-
|
39
|
-
|
40
|
-
|
39
|
+
a1 = assembly 'a' do
|
40
|
+
pass
|
41
|
+
end
|
41
42
|
|
42
|
-
|
43
|
-
|
44
|
-
|
43
|
+
a2 = assembly 'a' do
|
44
|
+
pass
|
45
|
+
end
|
45
46
|
|
46
|
-
|
47
|
-
|
47
|
+
x = assembly 'x' do
|
48
|
+
union 'a'
|
49
|
+
end
|
48
50
|
end
|
49
51
|
end
|
50
|
-
|
51
|
-
# FIXME: 'a' assemblies and qualified names collide
|
52
|
-
assert_equal 2, flow.children.size
|
53
|
-
assert_equal 'flow.a', a1.qualified_name
|
54
|
-
assert_equal 'flow.a', a2.qualified_name
|
55
|
-
|
56
|
-
# Ordered child names do not collide
|
57
|
-
assert_equal ['a', 'a', 'x'], flow.child_names
|
58
|
-
|
59
|
-
# FIXME: assembly defined last wins
|
60
|
-
assert_equal a2, flow.find_child('a')
|
61
|
-
|
62
|
-
assert_equal 1, x.tail_pipe.heads.size
|
63
|
-
assert_equal a2.head_pipe, x.tail_pipe.heads.first
|
52
|
+
assert_equal "Attempted to add 'flow.a', but node named 'a' already exists", ex.message
|
64
53
|
end
|
65
54
|
|
66
55
|
def test_ambiguous_branch_names
|
67
|
-
|
56
|
+
# You _can_ define ambiguously named branches
|
57
|
+
b1, b2 = nil, nil
|
68
58
|
flow = flow 'flow' do
|
69
59
|
source 'a', tap('test/data/data1.txt')
|
70
60
|
source 'b', tap('test/data/data1.txt')
|
@@ -79,21 +69,48 @@ class TC_Flow < Test::Unit::TestCase
|
|
79
69
|
pass
|
80
70
|
end
|
81
71
|
|
82
|
-
|
83
|
-
union 'b'
|
84
|
-
end
|
72
|
+
sink 'b', tap('output/test_ambiguous_branch_names')
|
85
73
|
end
|
86
74
|
|
87
|
-
|
88
|
-
assert_equal 3, flow.children.size
|
75
|
+
assert_equal 2, flow.children.size
|
89
76
|
assert_equal 'flow.a.b', b1.qualified_name
|
90
77
|
assert_equal 'flow.b', b2.qualified_name
|
91
|
-
assert_equal ['a', 'b'
|
78
|
+
assert_equal ['a', 'b'], flow.child_names
|
79
|
+
|
80
|
+
# You _cannot_ look them up using find_child
|
81
|
+
ex = assert_raise AmbiguousNodeNameException do
|
82
|
+
flow.find_child('b')
|
83
|
+
end
|
84
|
+
assert_equal "Ambiguous lookup of child by name 'b'; found 'flow.b', 'flow.a.b'", ex.message
|
85
|
+
|
86
|
+
# Which means you cannot sink them
|
87
|
+
ex = assert_raise AmbiguousNodeNameException do
|
88
|
+
flow.complete # NOTE: We must complete for sink to raise
|
89
|
+
end
|
90
|
+
assert_equal "Ambiguous lookup of child by name 'b'; found 'flow.b', 'flow.a.b'", ex.message
|
91
|
+
|
92
|
+
# And you cannot use them for join or union
|
93
|
+
ex = assert_raise AmbiguousNodeNameException do
|
94
|
+
b1, b2, x = nil, nil, nil
|
95
|
+
flow = flow 'flow' do
|
96
|
+
source 'a', tap('test/data/data1.txt')
|
97
|
+
source 'b', tap('test/data/data1.txt')
|
98
|
+
|
99
|
+
assembly 'a' do
|
100
|
+
b1 = branch 'b' do
|
101
|
+
pass
|
102
|
+
end
|
103
|
+
end
|
92
104
|
|
93
|
-
|
94
|
-
|
105
|
+
b2 = assembly 'b' do
|
106
|
+
pass
|
107
|
+
end
|
95
108
|
|
96
|
-
|
97
|
-
|
109
|
+
x = assembly 'x' do
|
110
|
+
union 'b'
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
assert_equal "Ambiguous lookup of child by name 'b'; found 'flow.b', 'flow.a.b'", ex.message
|
98
115
|
end
|
99
116
|
end
|
@@ -3,7 +3,7 @@ require 'cascading'
|
|
3
3
|
|
4
4
|
class TC_LocalExecution < Test::Unit::TestCase
|
5
5
|
def test_smoke_test_multi_source_tap
|
6
|
-
cascade 'splitter' do
|
6
|
+
cascade 'splitter', :mode => :local do
|
7
7
|
flow 'splitter' do
|
8
8
|
tap1 = tap 'test/data/data1.txt'
|
9
9
|
tap2 = tap 'test/data/data2.txt'
|
@@ -19,7 +19,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def test_smoke_test_sequence_file_scheme
|
22
|
-
cascade 'smoke' do
|
22
|
+
cascade 'smoke', :mode => :local do
|
23
23
|
flow 'smoke' do
|
24
24
|
source 'input', tap('test/data/data1.txt')
|
25
25
|
assembly 'input' do
|
@@ -32,7 +32,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def test_splitter
|
35
|
-
flow = flow 'splitter' do
|
35
|
+
flow = flow 'splitter', :mode => :local do
|
36
36
|
source 'copy', tap('test/data/data1.txt')
|
37
37
|
|
38
38
|
assembly 'copy' do
|
@@ -47,7 +47,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def test_smoke_test_multi_source_tap
|
50
|
-
cascade 'multi_source_tap' do
|
50
|
+
cascade 'multi_source_tap', :mode => :local do
|
51
51
|
flow 'multi_source_tap' do
|
52
52
|
tap1 = tap 'test/data/data1.txt'
|
53
53
|
tap2 = tap 'test/data/data2.txt'
|
@@ -64,7 +64,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
64
64
|
|
65
65
|
def test_join1
|
66
66
|
join_grouping_fields, join_values_fields = nil, nil
|
67
|
-
cascade 'splitter' do
|
67
|
+
cascade 'splitter', :mode => :local do
|
68
68
|
flow 'splitter' do
|
69
69
|
source 'data1', tap('test/data/data1.txt')
|
70
70
|
source 'data2', tap('test/data/data2.txt')
|
@@ -101,7 +101,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
101
101
|
|
102
102
|
def test_join2
|
103
103
|
join_grouping_fields, join_values_fields = nil, nil
|
104
|
-
flow = flow 'splitter' do
|
104
|
+
flow = flow 'splitter', :mode => :local do
|
105
105
|
source 'data1', tap('test/data/data1.txt')
|
106
106
|
source 'data2', tap('test/data/data2.txt')
|
107
107
|
|
@@ -129,7 +129,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
129
129
|
|
130
130
|
def test_union
|
131
131
|
union_grouping_fields, union_values_fields = nil, nil
|
132
|
-
cascade 'union' do
|
132
|
+
cascade 'union', :mode => :local do
|
133
133
|
flow 'union' do
|
134
134
|
source 'data1', tap('test/data/data1.txt')
|
135
135
|
source 'data2', tap('test/data/data2.txt')
|
data/test/test_operations.rb
CHANGED
@@ -13,4 +13,65 @@ class TC_Operations < Test::Unit::TestCase
|
|
13
13
|
first = first_function 'first_field', :ignore => [Java::CascadingTuple::Tuple.new(-1)].to_java(Java::CascadingTuple::Tuple)
|
14
14
|
assert_not_nil first
|
15
15
|
end
|
16
|
+
|
17
|
+
def test_coerce_to_java_int
|
18
|
+
result = coerce_to_java(1)
|
19
|
+
|
20
|
+
assert_equal Java::JavaLang::Long, result.class
|
21
|
+
assert_equal 1.to_java(java.lang.Long).java_class, result.java_class
|
22
|
+
assert_equal 1.to_java(java.lang.Long), result
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_coerce_to_java_long
|
26
|
+
result = coerce_to_java(32503701600000)
|
27
|
+
|
28
|
+
assert_equal Java::JavaLang::Long, result.class
|
29
|
+
assert_equal 32503701600000.to_java(java.lang.Long).java_class, result.java_class
|
30
|
+
assert_equal 32503701600000.to_java(java.lang.Long), result
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_coerce_to_java_string
|
34
|
+
result = coerce_to_java('a')
|
35
|
+
|
36
|
+
assert_equal Java::JavaLang::String, result.class
|
37
|
+
assert_equal 'a'.to_java(java.lang.String).java_class, result.java_class
|
38
|
+
assert_equal 'a'.to_java(java.lang.String), result
|
39
|
+
end
|
40
|
+
|
41
|
+
def test_coerce_to_java_float
|
42
|
+
result = coerce_to_java(5.4)
|
43
|
+
|
44
|
+
assert_equal Java::JavaLang::Double, result.class
|
45
|
+
assert_equal (5.4).to_java(java.lang.Double).java_class, result.java_class
|
46
|
+
assert_equal (5.4).to_java(java.lang.Double), result
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_coerce_to_java_double
|
50
|
+
result = coerce_to_java(10e100)
|
51
|
+
|
52
|
+
assert_equal Java::JavaLang::Double, result.class
|
53
|
+
assert_equal (10e100).to_java(java.lang.Double).java_class, result.java_class
|
54
|
+
assert_equal (10e100).to_java(java.lang.Double), result
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_coerce_to_java_nil
|
58
|
+
result = coerce_to_java(nil)
|
59
|
+
|
60
|
+
assert_equal NilClass, result.class
|
61
|
+
assert_nil result
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_coerce_to_java_other
|
65
|
+
result = coerce_to_java([1,2,3])
|
66
|
+
|
67
|
+
assert_equal Java::JavaLang::String, result.class
|
68
|
+
assert_equal ''.to_java(java.lang.String).java_class, result.java_class
|
69
|
+
assert_equal '123'.to_java(java.lang.String), result
|
70
|
+
end
|
71
|
+
|
72
|
+
def test_to_java_comparable_array
|
73
|
+
results = to_java_comparable_array([1, 'string', 1.5, nil])
|
74
|
+
|
75
|
+
assert_equal results.map{|i| i.class}, [Fixnum, String, Float, NilClass]
|
76
|
+
end
|
16
77
|
end
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: cascading.jruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.9
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Matt Walker
|
@@ -11,7 +11,7 @@ autorequire:
|
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
13
|
|
14
|
-
date: 2012-05
|
14
|
+
date: 2012-08-05 00:00:00 Z
|
15
15
|
dependencies: []
|
16
16
|
|
17
17
|
description: cascading.jruby is a small DSL above Cascading, written in JRuby
|
@@ -56,9 +56,11 @@ files:
|
|
56
56
|
- lib/cascading/expr_stub.rb
|
57
57
|
- lib/cascading/ext/array.rb
|
58
58
|
- lib/cascading/flow.rb
|
59
|
+
- lib/cascading/mode.rb
|
59
60
|
- lib/cascading/operations.rb
|
60
61
|
- lib/cascading/scope.rb
|
61
62
|
- lib/cascading/sub_assembly.rb
|
63
|
+
- lib/cascading/tap.rb
|
62
64
|
- samples/branch.rb
|
63
65
|
- samples/copy.rb
|
64
66
|
- samples/data/data2.txt
|