cascading.jruby 0.0.10 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE.txt +13 -160
- data/README.md +35 -0
- data/lib/cascading.rb +8 -41
- data/lib/cascading/aggregations.rb +216 -71
- data/lib/cascading/assembly.rb +409 -606
- data/lib/cascading/base.rb +22 -0
- data/lib/cascading/cascade.rb +55 -18
- data/lib/cascading/cascading.rb +137 -47
- data/lib/cascading/expr_stub.rb +31 -17
- data/lib/cascading/ext/array.rb +17 -0
- data/lib/cascading/filter_operations.rb +101 -0
- data/lib/cascading/flow.rb +87 -23
- data/lib/cascading/identity_operations.rb +82 -0
- data/lib/cascading/mode.rb +14 -10
- data/lib/cascading/operations.rb +109 -174
- data/lib/cascading/regex_operations.rb +133 -0
- data/lib/cascading/scope.rb +32 -9
- data/lib/cascading/sub_assembly.rb +8 -5
- data/lib/cascading/tap.rb +41 -17
- data/lib/cascading/text_operations.rb +67 -0
- data/test/mock_assemblies.rb +55 -0
- data/test/test_assembly.rb +23 -25
- data/test/test_local_execution.rb +7 -7
- data/test/test_operations.rb +0 -10
- metadata +76 -74
- data/History.txt +0 -58
@@ -4,17 +4,15 @@ module Cascading
|
|
4
4
|
# Allows you to plugin c.p.SubAssemblies to a cascading.jruby Assembly.
|
5
5
|
#
|
6
6
|
# Assumptions:
|
7
|
-
# * You will either use the tail_pipe of the calling Assembly, or overwrite
|
8
|
-
#
|
9
|
-
# * Your subassembly will have only 1 tail pipe; branching is not
|
10
|
-
# supported. This allows you to continue operating upon the tail of the
|
11
|
-
# SubAssembly within the calling Assembly
|
7
|
+
# * You will either use the tail_pipe of the calling Assembly, or overwrite its incoming_scopes (as do join and union)
|
8
|
+
# * Your subassembly will have only 1 tail pipe; branching is not supported. This allows you to continue operating upon the tail of the SubAssembly within the calling Assembly
|
12
9
|
# * You will not use nested c.p.SubAssemblies
|
13
10
|
#
|
14
11
|
# This is a low-level tool, so be careful.
|
15
12
|
class SubAssembly
|
16
13
|
attr_reader :assembly, :sub_assembly, :tail_pipe, :scope
|
17
14
|
|
15
|
+
# Construct a SubAssembly within the given Assembly
|
18
16
|
def initialize(assembly, sub_assembly)
|
19
17
|
@assembly = assembly
|
20
18
|
@sub_assembly = sub_assembly
|
@@ -26,6 +24,11 @@ module Cascading
|
|
26
24
|
raise 'SubAssembly must set exactly 1 tail in constructor' unless sub_assembly.tails.size == 1
|
27
25
|
end
|
28
26
|
|
27
|
+
# Complete the addition of the SubAssembly to the Assembly. Propagates
|
28
|
+
# Scope through the SubAssembly and updates the tail_pipe of the
|
29
|
+
# SubAssembly for passing back to the enclosing Assembly. May accept many
|
30
|
+
# incoming pipes, but typically only recieves the tail_pipe of the
|
31
|
+
# enclosing Assembly.
|
29
32
|
def finalize(pipes, incoming_scopes)
|
30
33
|
# Build adjacency list for sub_assembly
|
31
34
|
graph = {}
|
data/lib/cascading/tap.rb
CHANGED
@@ -1,48 +1,63 @@
|
|
1
1
|
module Cascading
|
2
|
-
# A
|
3
|
-
#
|
2
|
+
# A BaseTap wraps up a pair of Cascading taps, one for Cascading local mode
|
3
|
+
# and the other for Hadoop mode. Note that these are optional, but at least
|
4
|
+
# one must be provided for most taps. A SequenceFile is a notable example of
|
5
|
+
# a Scheme for which there is no Cascading local mode version, so a Tap you
|
6
|
+
# build with it will have no local_tap.
|
4
7
|
class BaseTap
|
5
8
|
attr_reader :local_tap, :hadoop_tap
|
6
9
|
|
10
|
+
# Constructor that accepts the local_tap and hadoop_tap, which may be nil
|
7
11
|
def initialize(local_tap, hadoop_tap)
|
8
12
|
@local_tap = local_tap
|
9
13
|
@hadoop_tap = hadoop_tap
|
10
14
|
end
|
11
15
|
|
16
|
+
# Passes through printing the local_tap and hadoop_tap
|
12
17
|
def to_s
|
13
18
|
"Local: #{local_tap}, Hadoop: #{hadoop_tap}"
|
14
19
|
end
|
15
20
|
|
21
|
+
# Returns false if the local_tap is nil, true otherwise
|
16
22
|
def local?
|
17
23
|
!local_tap.nil?
|
18
24
|
end
|
19
25
|
|
26
|
+
# Returns false if the hadoop_tap is nil, true otherwise
|
20
27
|
def hadoop?
|
21
28
|
!hadoop_tap.nil?
|
22
29
|
end
|
23
30
|
end
|
24
31
|
|
25
|
-
# A
|
26
|
-
#
|
27
|
-
#
|
28
|
-
#
|
29
|
-
# :
|
30
|
-
#
|
31
|
-
# mode.
|
32
|
+
# A Tap represents a non-aggregate tap with a scheme, path, and optional
|
33
|
+
# sink_mode. c.t.l.FileTap is used in Cascading local mode and c.t.h.Hfs is
|
34
|
+
# used in Hadoop mode. Whether or not these can be created is governed by the
|
35
|
+
# :scheme parameter, which must contain at least one of :local_scheme or
|
36
|
+
# :hadoop_scheme. Schemes like TextLine are supported in both modes (by
|
37
|
+
# Cascading), but SequenceFile is only supported in Hadoop mode.
|
32
38
|
class Tap < BaseTap
|
33
39
|
attr_reader :scheme, :path, :sink_mode
|
34
40
|
|
35
|
-
|
41
|
+
# Builds a Tap given a required path
|
42
|
+
#
|
43
|
+
# The named options are:
|
44
|
+
# [scheme] A Hash which must contain at least one of :local_scheme or
|
45
|
+
# :hadoop_scheme but may contain both. Default is
|
46
|
+
# text_line_scheme, which works in both modes.
|
47
|
+
# [sink_mode] A symbol or string that may be :keep, :replace, or :append,
|
48
|
+
# and corresponds to the c.t.SinkMode enumeration. The default
|
49
|
+
# is :keep, which matches Cascading's default.
|
50
|
+
def initialize(path, options = {})
|
36
51
|
@path = path
|
37
52
|
|
38
|
-
@scheme =
|
53
|
+
@scheme = options[:scheme] || text_line_scheme
|
39
54
|
raise "Scheme must provide one of :local_scheme or :hadoop_scheme; received: '#{scheme.inspect}'" unless scheme[:local_scheme] || scheme[:hadoop_scheme]
|
40
55
|
|
41
|
-
@sink_mode = case
|
56
|
+
@sink_mode = case options[:sink_mode] || :keep
|
42
57
|
when :keep, 'keep' then Java::CascadingTap::SinkMode::KEEP
|
43
58
|
when :replace, 'replace' then Java::CascadingTap::SinkMode::REPLACE
|
44
59
|
when :append, 'append' then Java::CascadingTap::SinkMode::APPEND
|
45
|
-
else raise "Unrecognized sink mode '#{
|
60
|
+
else raise "Unrecognized sink mode '#{options[:sink_mode]}'"
|
46
61
|
end
|
47
62
|
|
48
63
|
local_scheme = scheme[:local_scheme]
|
@@ -53,19 +68,28 @@ module Cascading
|
|
53
68
|
end
|
54
69
|
end
|
55
70
|
|
56
|
-
# A
|
57
|
-
#
|
58
|
-
#
|
59
|
-
#
|
71
|
+
# A MultiTap represents one of Cascading's aggregate taps and is built via
|
72
|
+
# static constructors that accept an array of Taps. In order for a mode
|
73
|
+
# (Cascading local or Hadoop) to be supported, all provided taps must support
|
74
|
+
# it.
|
60
75
|
class MultiTap < BaseTap
|
76
|
+
# Do not call this constructor directly; instead, use one of
|
77
|
+
# MultiTap.multi_source_tap or MultiTap.multi_sink_tap.
|
61
78
|
def initialize(local_tap, hadoop_tap)
|
62
79
|
super(local_tap, hadoop_tap)
|
63
80
|
end
|
64
81
|
|
82
|
+
# Static constructor that builds a MultiTap wrapping a c.t.MultiSourceTap
|
83
|
+
# from the given array of Taps. The resulting MultiTap will only be
|
84
|
+
# available in Cascading local mode or Hadoop mode if all input taps support
|
85
|
+
# them.
|
65
86
|
def self.multi_source_tap(taps)
|
66
87
|
multi_tap(taps, Java::CascadingTap::MultiSourceTap)
|
67
88
|
end
|
68
89
|
|
90
|
+
# Static constructor that builds a MultiTap wrapping a c.t.MultiSinkTap from
|
91
|
+
# the given array of Taps. The resulting MultiTap will only be available in
|
92
|
+
# Cascading local mode or Hadoop mode if all input taps support them.
|
69
93
|
def self.multi_sink_tap(taps)
|
70
94
|
multi_tap(taps, Java::CascadingTap::MultiSinkTap)
|
71
95
|
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Cascading
|
2
|
+
# Module of pipe assemblies that wrap operations defined in the Cascading
|
3
|
+
# cascading.operations.text package. These are split out only to group
|
4
|
+
# similar functionality.
|
5
|
+
#
|
6
|
+
# Mapping of DSL pipes into Cascading text operations:
|
7
|
+
# parse\_date:: {DateParser}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/text/DateParser.html]
|
8
|
+
# format\_date:: {DateFormatter}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/text/DateFormatter.html]
|
9
|
+
# join\_fields:: {FieldJoiner}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/text/FieldJoiner.html]
|
10
|
+
module TextOperations
|
11
|
+
# Parses the given input_field as a date using the provided format string.
|
12
|
+
#
|
13
|
+
# Example:
|
14
|
+
# parse_date 'text_date', 'yyyy/MM/dd', 'timestamp'
|
15
|
+
def parse_date(input_field, date_format, into_field, options = {})
|
16
|
+
output = options[:output] || all_fields # Overrides Cascading default
|
17
|
+
|
18
|
+
input_field = fields(input_field)
|
19
|
+
raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
|
20
|
+
into_field = fields(into_field)
|
21
|
+
raise "into_field must declare exactly one field, was '#{into_field}'" unless into_field.size == 1
|
22
|
+
|
23
|
+
each(
|
24
|
+
input_field,
|
25
|
+
:function => Java::CascadingOperationText::DateParser.new(into_field, date_format),
|
26
|
+
:output => output
|
27
|
+
)
|
28
|
+
end
|
29
|
+
|
30
|
+
# Converts a timestamp into a formatted date string using the specified
|
31
|
+
# date_format.
|
32
|
+
#
|
33
|
+
# Example:
|
34
|
+
# format_date 'timestamp', 'yyyy/MM/dd', 'text_date'
|
35
|
+
def format_date(input_field, date_format, into_field, options = {})
|
36
|
+
output = options[:output] || all_fields # Overrides Cascading default
|
37
|
+
|
38
|
+
input_field = fields(input_field)
|
39
|
+
raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
|
40
|
+
into_field = fields(into_field)
|
41
|
+
raise "into_field must declare exactly one field, was '#{into_field}'" unless into_field.size == 1
|
42
|
+
|
43
|
+
each(
|
44
|
+
input_field,
|
45
|
+
:function => Java::CascadingOperationText::DateFormatter.new(into_field, date_format),
|
46
|
+
:output => output
|
47
|
+
)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Joins multiple fields into a single field given a delimiter.
|
51
|
+
#
|
52
|
+
# Example:
|
53
|
+
# join_fields ['field1', 'field2'], ',', 'comma_separated'
|
54
|
+
def join_fields(input_fields, delimiter, into_field)
|
55
|
+
output = options[:output] || all_fields # Overrides Cascading default
|
56
|
+
|
57
|
+
into_field = fields(into_field)
|
58
|
+
raise "into_field must declare exactly one field, was '#{into_field}'" unless into_field.size == 1
|
59
|
+
|
60
|
+
each(
|
61
|
+
input_fields,
|
62
|
+
:function => Java::CascadingOperationText::FieldJoiner.new(into_field, delimiter.to_s),
|
63
|
+
:output => output
|
64
|
+
)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'cascading'
|
2
|
+
|
3
|
+
module MockAssemblies
|
4
|
+
def mock_assembly(&block)
|
5
|
+
assembly = nil
|
6
|
+
flow 'test' do
|
7
|
+
source 'test', tap('test/data/data1.txt')
|
8
|
+
assembly = assembly 'test', &block
|
9
|
+
sink 'test', tap('output/test_mock_assembly')
|
10
|
+
end
|
11
|
+
assembly
|
12
|
+
end
|
13
|
+
|
14
|
+
def mock_branched_assembly(&block)
|
15
|
+
assembly = nil
|
16
|
+
flow 'mock_branched_assembly' do
|
17
|
+
source 'data1', tap('test/data/data1.txt')
|
18
|
+
|
19
|
+
assembly 'data1' do
|
20
|
+
branch 'test1' do
|
21
|
+
pass
|
22
|
+
end
|
23
|
+
branch 'test2' do
|
24
|
+
pass
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
assembly = assembly 'test', &block
|
29
|
+
|
30
|
+
sink 'test', tap('output/test_mock_branched_assembly')
|
31
|
+
end
|
32
|
+
assembly
|
33
|
+
end
|
34
|
+
|
35
|
+
def mock_two_input_assembly(&block)
|
36
|
+
assembly = nil
|
37
|
+
flow 'mock_two_input_assembly' do
|
38
|
+
source 'test1', tap('test/data/data1.txt')
|
39
|
+
source 'test2', tap('test/data/data2.txt')
|
40
|
+
|
41
|
+
assembly 'test1' do
|
42
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
43
|
+
end
|
44
|
+
|
45
|
+
assembly 'test2' do
|
46
|
+
split 'line', /[.,]*\s+/, ['name', 'id', 'town'], :output => ['name', 'id', 'town']
|
47
|
+
end
|
48
|
+
|
49
|
+
assembly = assembly 'test', &block
|
50
|
+
|
51
|
+
sink 'test', tap('output/test_mock_two_input_assembly')
|
52
|
+
end
|
53
|
+
assembly
|
54
|
+
end
|
55
|
+
end
|
data/test/test_assembly.rb
CHANGED
@@ -23,7 +23,7 @@ class TC_Assembly < Test::Unit::TestCase
|
|
23
23
|
|
24
24
|
def test_each_identity
|
25
25
|
assembly = mock_assembly do
|
26
|
-
each 'offset', :function =>
|
26
|
+
each 'offset', :function => Java::CascadingOperation::Identity.new
|
27
27
|
end
|
28
28
|
|
29
29
|
flow = assembly.parent
|
@@ -35,7 +35,7 @@ class TC_Assembly < Test::Unit::TestCase
|
|
35
35
|
def test_create_each
|
36
36
|
# You can apply an Each to 0 fields
|
37
37
|
assembly = mock_assembly do
|
38
|
-
each(:function =>
|
38
|
+
each(:function => Java::CascadingOperation::Identity.new)
|
39
39
|
end
|
40
40
|
assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
|
41
41
|
|
@@ -547,7 +547,7 @@ class TC_Assembly < Test::Unit::TestCase
|
|
547
547
|
end
|
548
548
|
end
|
549
549
|
end
|
550
|
-
assert_equal "
|
550
|
+
assert_equal "HashJoin doesn't support aggregations so the block provided to hash_join will be ignored", ex.message
|
551
551
|
end
|
552
552
|
|
553
553
|
def test_branch_unique
|
@@ -579,7 +579,7 @@ class TC_Assembly < Test::Unit::TestCase
|
|
579
579
|
assembly = mock_assembly do
|
580
580
|
branch 'branch1' do
|
581
581
|
branch 'branch2' do
|
582
|
-
each 'line', :function =>
|
582
|
+
each 'line', :function => Java::CascadingOperation::Identity.new
|
583
583
|
end
|
584
584
|
end
|
585
585
|
end
|
@@ -659,43 +659,41 @@ class TC_Assembly < Test::Unit::TestCase
|
|
659
659
|
assert_equal ['line', 'sum'], assembly.scope.grouping_fields.to_a
|
660
660
|
end
|
661
661
|
|
662
|
-
def
|
662
|
+
def test_where
|
663
663
|
assembly = mock_assembly do
|
664
|
-
split 'line', ['name', 'score1', 'score2', 'id'], :
|
665
|
-
where
|
664
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
665
|
+
where 'score1:double < score2:double'
|
666
666
|
end
|
667
667
|
assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
|
668
|
-
|
669
|
-
# Empty where compiles away
|
670
|
-
assert_equal Java::CascadingOperationRegex::RegexSplitter, assembly.tail_pipe.operation.class
|
668
|
+
assert_equal Java::CascadingOperationExpression::ExpressionFilter, assembly.tail_pipe.operation.class
|
671
669
|
end
|
672
670
|
|
673
|
-
def
|
671
|
+
def test_where_with_import
|
674
672
|
assembly = mock_assembly do
|
675
|
-
split 'line', ['name', 'score1', 'score2', 'id'], :
|
676
|
-
|
673
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
674
|
+
names = ['SMITH', 'JONES', 'BROWN']
|
675
|
+
where "import java.util.Arrays;\nArrays.asList(new String[] { \"#{names.join('", "')}\" }).contains(name:string)"
|
677
676
|
end
|
678
677
|
assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
|
679
678
|
assert_equal Java::CascadingOperationExpression::ExpressionFilter, assembly.tail_pipe.operation.class
|
680
679
|
end
|
681
680
|
|
682
|
-
def
|
681
|
+
def test_rename
|
683
682
|
assembly = mock_assembly do
|
684
|
-
split 'line', ['name', 'score1', 'score2', 'id'], :
|
685
|
-
|
683
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
684
|
+
rename 'score2' => 'new_score2', 'score1' => 'new_score1', 'name' => 'new_name'
|
686
685
|
end
|
687
|
-
|
688
|
-
assert_equal
|
686
|
+
# Original order preserved
|
687
|
+
assert_equal ['new_name', 'new_score1', 'new_score2', 'id'], assembly.scope.values_fields.to_a
|
689
688
|
end
|
690
689
|
|
691
|
-
def
|
690
|
+
def test_copy
|
692
691
|
assembly = mock_assembly do
|
693
|
-
split 'line', ['name', 'score1', 'score2', 'id'], :
|
694
|
-
|
695
|
-
where "import java.util.Arrays;\nArrays.asList(new String[] { \"#{names.join('", "')}\" }).contains(name:string)"
|
692
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
693
|
+
copy 'score2' => 'new_score2', 'id' => 'new_id', 'name' => 'new_name'
|
696
694
|
end
|
697
|
-
|
698
|
-
assert_equal
|
695
|
+
# Original order preserved in copied fields
|
696
|
+
assert_equal ['name', 'score1', 'score2', 'id', 'new_name', 'new_score2', 'new_id'], assembly.scope.values_fields.to_a
|
699
697
|
end
|
700
698
|
|
701
699
|
def test_smoke_test_describe
|
@@ -711,7 +709,7 @@ class TC_Assembly < Test::Unit::TestCase
|
|
711
709
|
end
|
712
710
|
puts "Describe at assembly end: '#{describe}'"
|
713
711
|
end
|
714
|
-
sink 'input', tap('output/
|
712
|
+
sink 'input', tap('output/test_smoke_test_describe')
|
715
713
|
end
|
716
714
|
end
|
717
715
|
end
|
@@ -36,7 +36,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
36
36
|
source 'copy', tap('test/data/data1.txt')
|
37
37
|
|
38
38
|
assembly 'copy' do
|
39
|
-
split 'line',
|
39
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
40
40
|
assert_size_equals 4
|
41
41
|
assert_not_null
|
42
42
|
debug :print_fields => true
|
@@ -70,14 +70,14 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
70
70
|
source 'data2', tap('test/data/data2.txt')
|
71
71
|
|
72
72
|
assembly1 = assembly 'data1' do
|
73
|
-
split 'line',
|
73
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
74
74
|
assert_size_equals 4
|
75
75
|
assert_not_null
|
76
76
|
debug :print_fields => true
|
77
77
|
end
|
78
78
|
|
79
79
|
assembly2 = assembly 'data2' do
|
80
|
-
split 'line',
|
80
|
+
split 'line', /[.,]*\s+/, ['name', 'id', 'town'], :output => ['name', 'id', 'town']
|
81
81
|
assert_size_equals 3
|
82
82
|
assert_not_null
|
83
83
|
debug :print_fields => true
|
@@ -106,12 +106,12 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
106
106
|
source 'data2', tap('test/data/data2.txt')
|
107
107
|
|
108
108
|
assembly 'data1' do
|
109
|
-
split 'line',
|
109
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
110
110
|
debug :print_fields => true
|
111
111
|
end
|
112
112
|
|
113
113
|
assembly 'data2' do
|
114
|
-
split 'line',
|
114
|
+
split 'line', /[.,]*\s+/, ['name', 'code', 'town'], :output => ['name', 'code', 'town']
|
115
115
|
debug :print_fields => true
|
116
116
|
end
|
117
117
|
|
@@ -135,7 +135,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
135
135
|
source 'data2', tap('test/data/data2.txt')
|
136
136
|
|
137
137
|
assembly 'data1' do
|
138
|
-
split 'line',
|
138
|
+
split 'line', /[.,]*\s+/, ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
|
139
139
|
assert_size_equals 4
|
140
140
|
assert_not_null
|
141
141
|
|
@@ -144,7 +144,7 @@ class TC_LocalExecution < Test::Unit::TestCase
|
|
144
144
|
end
|
145
145
|
|
146
146
|
assembly 'data2' do
|
147
|
-
split 'line',
|
147
|
+
split 'line', /[.,]*\s+/, ['name', 'code', 'town'], :output => ['name', 'code', 'town']
|
148
148
|
assert_size_equals 3
|
149
149
|
assert_not_null
|
150
150
|
|
data/test/test_operations.rb
CHANGED
@@ -4,16 +4,6 @@ require 'cascading'
|
|
4
4
|
class TC_Operations < Test::Unit::TestCase
|
5
5
|
include Operations
|
6
6
|
|
7
|
-
def test_aggregator_function_ignore_values
|
8
|
-
min = min_function 'min_field', :ignore => [nil].to_java(:string)
|
9
|
-
assert_not_nil min
|
10
|
-
end
|
11
|
-
|
12
|
-
def test_aggregator_function_ignore_tuples
|
13
|
-
first = first_function 'first_field', :ignore => [Java::CascadingTuple::Tuple.new(-1)].to_java(Java::CascadingTuple::Tuple)
|
14
|
-
assert_not_nil first
|
15
|
-
end
|
16
|
-
|
17
7
|
def test_coerce_to_java_int
|
18
8
|
result = coerce_to_java(1)
|
19
9
|
|
metadata
CHANGED
@@ -1,88 +1,90 @@
|
|
1
|
-
--- !ruby/object:Gem::Specification
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
2
|
name: cascading.jruby
|
3
|
-
version: !ruby/object:Gem::Version
|
4
|
-
prerelease:
|
5
|
-
version: 0.0
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 1.0.0
|
6
6
|
platform: ruby
|
7
|
-
authors:
|
8
|
-
|
9
|
-
|
10
|
-
autorequire:
|
7
|
+
authors:
|
8
|
+
- Matt Walker
|
9
|
+
- Grégoire Marabout
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
|
14
|
-
date: 2012-08-09 00:00:00 Z
|
13
|
+
date: 2013-04-26 00:00:00.000000000 Z
|
15
14
|
dependencies: []
|
16
|
-
|
17
15
|
description: cascading.jruby is a small DSL above Cascading, written in JRuby
|
18
|
-
email:
|
16
|
+
email: matt.r.walker@gmail.com
|
19
17
|
executables: []
|
20
|
-
|
21
18
|
extensions: []
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
19
|
+
extra_rdoc_files:
|
20
|
+
- README.md
|
21
|
+
- LICENSE.txt
|
22
|
+
files:
|
23
|
+
- lib/cascading.rb
|
24
|
+
- lib/cascading/aggregations.rb
|
25
|
+
- lib/cascading/assembly.rb
|
26
|
+
- lib/cascading/base.rb
|
27
|
+
- lib/cascading/cascade.rb
|
28
|
+
- lib/cascading/cascading.rb
|
29
|
+
- lib/cascading/cascading_exception.rb
|
30
|
+
- lib/cascading/expr_stub.rb
|
31
|
+
- lib/cascading/filter_operations.rb
|
32
|
+
- lib/cascading/flow.rb
|
33
|
+
- lib/cascading/identity_operations.rb
|
34
|
+
- lib/cascading/mode.rb
|
35
|
+
- lib/cascading/operations.rb
|
36
|
+
- lib/cascading/regex_operations.rb
|
37
|
+
- lib/cascading/scope.rb
|
38
|
+
- lib/cascading/sub_assembly.rb
|
39
|
+
- lib/cascading/tap.rb
|
40
|
+
- lib/cascading/text_operations.rb
|
41
|
+
- lib/cascading/ext/array.rb
|
42
|
+
- README.md
|
43
|
+
- LICENSE.txt
|
44
|
+
- test/mock_assemblies.rb
|
45
|
+
- test/test_aggregations.rb
|
46
|
+
- test/test_assembly.rb
|
47
|
+
- test/test_cascade.rb
|
48
|
+
- test/test_cascading.rb
|
49
|
+
- test/test_exceptions.rb
|
50
|
+
- test/test_flow.rb
|
51
|
+
- test/test_local_execution.rb
|
52
|
+
- test/test_operations.rb
|
53
|
+
homepage: http://github.com/mrwalker/cascading.jruby
|
53
54
|
licenses: []
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
55
|
+
post_install_message:
|
56
|
+
rdoc_options:
|
57
|
+
- "--main"
|
58
|
+
- README.md
|
59
|
+
require_paths:
|
60
|
+
- lib
|
61
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
62
|
+
requirements:
|
63
|
+
- - ">="
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: !binary |-
|
66
|
+
MA==
|
62
67
|
none: false
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
69
|
+
requirements:
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: !binary |-
|
73
|
+
MA==
|
68
74
|
none: false
|
69
|
-
requirements:
|
70
|
-
- - ">="
|
71
|
-
- !ruby/object:Gem::Version
|
72
|
-
version: "0"
|
73
75
|
requirements: []
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
signing_key:
|
76
|
+
rubyforge_project:
|
77
|
+
rubygems_version: 1.8.24
|
78
|
+
signing_key:
|
78
79
|
specification_version: 3
|
79
80
|
summary: A JRuby DSL for Cascading
|
80
|
-
test_files:
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
81
|
+
test_files:
|
82
|
+
- test/mock_assemblies.rb
|
83
|
+
- test/test_aggregations.rb
|
84
|
+
- test/test_assembly.rb
|
85
|
+
- test/test_cascade.rb
|
86
|
+
- test/test_cascading.rb
|
87
|
+
- test/test_exceptions.rb
|
88
|
+
- test/test_flow.rb
|
89
|
+
- test/test_local_execution.rb
|
90
|
+
- test/test_operations.rb
|