zillabyte 0.1.43 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ruby/lib/zillabyte/harness/app.rb +7 -43
- data/ruby/lib/zillabyte/harness/common_node.rb +16 -0
- data/ruby/lib/zillabyte/harness/component.rb +5 -14
- data/ruby/lib/zillabyte/harness/component_sink.rb +38 -18
- data/ruby/lib/zillabyte/harness/component_source.rb +29 -9
- data/ruby/lib/zillabyte/harness/component_stream.rb +6 -13
- data/ruby/lib/zillabyte/harness/each.rb +44 -22
- data/ruby/lib/zillabyte/harness/filter.rb +40 -17
- data/ruby/lib/zillabyte/harness/group_by.rb +42 -22
- data/ruby/lib/zillabyte/harness/helper.rb +3 -1
- data/ruby/lib/zillabyte/harness/injected_component.rb +43 -19
- data/ruby/lib/zillabyte/harness/join.rb +59 -31
- data/ruby/lib/zillabyte/harness/jvm_operation.rb +28 -0
- data/ruby/lib/zillabyte/harness/operation_handler.rb +109 -0
- data/ruby/lib/zillabyte/harness/sink.rb +37 -18
- data/ruby/lib/zillabyte/harness/source.rb +60 -31
- data/ruby/lib/zillabyte/harness/stream.rb +92 -431
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +7 -4
@@ -1,31 +1,55 @@
|
|
1
1
|
class Zillabyte::Harness::InjectedComponent
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node, :_id, :_input_stream_1
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
@_type = "component"
|
7
|
-
@_consumes = []
|
8
|
-
@_output_format = :replace
|
9
|
-
end
|
4
|
+
class Node
|
5
|
+
attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
|
10
6
|
|
11
|
-
|
12
|
-
|
13
|
-
|
7
|
+
def initialize(input_stream_1)
|
8
|
+
@_name = "component_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = "component"
|
10
|
+
@_consumes = [input_stream_1]
|
11
|
+
@_output_format = :replace
|
12
|
+
end
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
def name(v)
|
15
|
+
@_name = v
|
16
|
+
end
|
17
|
+
|
18
|
+
def component_id(v)
|
19
|
+
@_id = v.to_s
|
20
|
+
end
|
18
21
|
|
19
|
-
|
20
|
-
|
22
|
+
def additional_inputs(*v)
|
23
|
+
@_consumes.push(*v)
|
24
|
+
end
|
25
|
+
|
26
|
+
def outputs(*v)
|
27
|
+
@_emits = *v
|
28
|
+
end
|
29
|
+
|
30
|
+
def output_format(v)
|
31
|
+
@_output_format = v
|
32
|
+
end
|
21
33
|
end
|
22
34
|
|
23
|
-
def
|
24
|
-
@
|
35
|
+
def initialize(app, *args)
|
36
|
+
@_app = app
|
37
|
+
@_input_stream_1 = args[0]
|
38
|
+
@_id = args[1]
|
25
39
|
end
|
26
40
|
|
27
|
-
def
|
28
|
-
@
|
41
|
+
def build_node(&block)
|
42
|
+
@_node = Node.new @_input_stream_1
|
43
|
+
@_node.instance_eval(&block) if block_given?
|
44
|
+
@_node._id ||= @_id
|
45
|
+
@_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()] # We default to single-stream when called like this..
|
46
|
+
|
47
|
+
Zillabyte::Harness::Helper.check_name("component", @_node._name, @_app._names)
|
48
|
+
Zillabyte::Harness::Helper.check_call_component(@_node)
|
49
|
+
Zillabyte::Harness::Helper.check_emits("component", @_node._emits, @_app._streams)
|
29
50
|
end
|
30
51
|
|
52
|
+
def run_operation
|
53
|
+
puts "Components can only be instantiated on our servers, sorry!"
|
54
|
+
end
|
31
55
|
end
|
@@ -1,39 +1,67 @@
|
|
1
1
|
class Zillabyte::Harness::Join
|
2
|
-
attr_accessor :
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
2
|
+
attr_accessor :_app, :_node, :_args
|
3
|
+
|
4
|
+
class Node
|
5
|
+
attr_accessor :_name, :_type, :_lhs_stream, :_rhs_stream, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
|
6
|
+
|
7
|
+
def initialize(*args)
|
8
|
+
@_name = "join_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = 'join'
|
10
|
+
@_join_type = :inner
|
11
|
+
@_lhs_stream = args[0]
|
12
|
+
@_rhs_stream = args[1]
|
13
|
+
options = args[2]
|
14
|
+
|
15
|
+
options.symbolize_keys!
|
16
|
+
|
17
|
+
emits = options[:emits]
|
18
|
+
if emits
|
19
|
+
if emits.is_a?(String)
|
20
|
+
@_emits = [options[:emits]]
|
21
|
+
else
|
22
|
+
@_emits = options[:emits]
|
23
|
+
end
|
18
24
|
end
|
19
|
-
end
|
20
25
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
26
|
+
# Figure out which fields to join on..
|
27
|
+
on = options[:on]
|
28
|
+
if on
|
29
|
+
if on.is_a?(Array) and on.size == 2
|
30
|
+
@_lhs_fields = on[0]
|
31
|
+
@_rhs_fields = on[1]
|
32
|
+
elsif (on.is_a?(String) or on.is_a?(Symbol))
|
33
|
+
@_lhs_fields = on
|
34
|
+
@_rhs_fields = on
|
35
|
+
end
|
30
36
|
end
|
31
|
-
end
|
32
37
|
|
33
|
-
|
34
|
-
|
35
|
-
|
38
|
+
# Different kind of join types...
|
39
|
+
if options[:type]
|
40
|
+
@_join_type = options[:type].to_sym
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def initialize(app, *args)
|
46
|
+
@_app = app
|
47
|
+
@_args = args
|
48
|
+
end
|
49
|
+
|
50
|
+
def build_node
|
51
|
+
@_node = Node.new(*@_args)
|
52
|
+
Zillabyte::Harness::Helper.check_name("join", @_node._name, @_app._names)
|
53
|
+
|
54
|
+
# Build the node
|
55
|
+
if @_node._emits
|
56
|
+
Zillabyte::Harness::Helper.check_emits("join", @_node._emits, @_app._streams)
|
57
|
+
else
|
58
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
36
59
|
end
|
37
|
-
|
60
|
+
|
61
|
+
Zillabyte::Harness::Helper.check_join(@_node)
|
62
|
+
end
|
63
|
+
|
64
|
+
def run_operation
|
65
|
+
puts "Join operations can only be instantiated on our cluster, sorry!"
|
38
66
|
end
|
39
67
|
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
class Zillabyte::Harness::JvmOperation
|
2
|
+
attr_accessor :_type, :_node, :_args
|
3
|
+
|
4
|
+
class Node
|
5
|
+
attr_accessor :_name, :_type, :_emits
|
6
|
+
|
7
|
+
def initialize(type)
|
8
|
+
@_name = "#{type}_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = type
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
13
|
+
|
14
|
+
def initialize(type, *args)
|
15
|
+
@_type = type
|
16
|
+
@_args = args
|
17
|
+
end
|
18
|
+
|
19
|
+
def build_node
|
20
|
+
@_node = Node.new @_type
|
21
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
22
|
+
@_node
|
23
|
+
end
|
24
|
+
|
25
|
+
def run_operation
|
26
|
+
puts "#{@_type} operation can only be instantiated on our cluster, sorry!"
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
class Zillabyte::Harness::OperationHandler
|
2
|
+
attr_accessor :_app, :_stream_class, :_operation, :_node_hash_elements, :_node_hash_pars, :_node_hash_optpars, :_arc_hash_streams
|
3
|
+
|
4
|
+
def initialize(app, stream_class)
|
5
|
+
@_app = app
|
6
|
+
@_stream_class = stream_class
|
7
|
+
@_node_hash_elements = {}
|
8
|
+
@_node_hash_pars = []
|
9
|
+
@_node_hash_optpars = []
|
10
|
+
@_arc_hash_streams = {}
|
11
|
+
end
|
12
|
+
|
13
|
+
def node
|
14
|
+
@_operation._node
|
15
|
+
end
|
16
|
+
|
17
|
+
def build_multilang_operation(type, *args, &block)
|
18
|
+
case type
|
19
|
+
when "source"
|
20
|
+
@_operation = Zillabyte::Harness::Source.new(@_app, *args)
|
21
|
+
when "each"
|
22
|
+
@_operation = Zillabyte::Harness::Each.new(@_app, *args)
|
23
|
+
when "filter"
|
24
|
+
@_operation = Zillabyte::Harness::Filter.new(@_app, *args)
|
25
|
+
when "group_by"
|
26
|
+
@_operation = Zillabyte::Harness::GroupBy.new(@_app, *args)
|
27
|
+
when "join"
|
28
|
+
@_operation = Zillabyte::Harness::Join.new(@_app, *args)
|
29
|
+
when "component"
|
30
|
+
@_operation = Zillabyte::Harness::InjectedComponent.new(@_app, *args)
|
31
|
+
when "sink"
|
32
|
+
@_operation = Zillabyte::Harness::Sink.new(@_app, *args)
|
33
|
+
when "component_source"
|
34
|
+
@_operation = Zillabyte::Harness::ComponentSource.new(@_app, *args)
|
35
|
+
when "component_sink"
|
36
|
+
@_operation = Zillabyte::Harness::ComponentSink.new(@_app, *args)
|
37
|
+
else
|
38
|
+
throw "Unknown type"
|
39
|
+
end
|
40
|
+
@_operation.build_node(&block)
|
41
|
+
self
|
42
|
+
end
|
43
|
+
|
44
|
+
def build_jvm_operation(type, *args)
|
45
|
+
@_operation = Zillabyte::Harness::JvmOperation.new(type, *args)
|
46
|
+
@_operation.build_node
|
47
|
+
self
|
48
|
+
end
|
49
|
+
|
50
|
+
def add_operation_properties_to_info(*args)
|
51
|
+
@_node_hash_pars.concat(args)
|
52
|
+
self
|
53
|
+
end
|
54
|
+
|
55
|
+
def add_optional_operation_properties_to_info(*args)
|
56
|
+
@_node_hash_optpars.concat(args)
|
57
|
+
self
|
58
|
+
end
|
59
|
+
|
60
|
+
def add_input_args_to_info_as(key, index = nil)
|
61
|
+
if !index.nil?
|
62
|
+
@_node_hash_elements[key] = @_operation._args[index]
|
63
|
+
else
|
64
|
+
@_node_hash_elements[key] = @_operation._args
|
65
|
+
end
|
66
|
+
self
|
67
|
+
end
|
68
|
+
|
69
|
+
def create_arc_info_from_stream(stream, direction = nil)
|
70
|
+
@_arc_hash_streams[stream] = direction
|
71
|
+
self
|
72
|
+
end
|
73
|
+
|
74
|
+
def get_output_streams(&block)
|
75
|
+
output_streams = []
|
76
|
+
node._emits.each do |stream|
|
77
|
+
output_streams << @_stream_class.new(stream, @_app, node._name)
|
78
|
+
end
|
79
|
+
output_streams = output_streams[0] if output_streams.size == 1
|
80
|
+
output_streams
|
81
|
+
end
|
82
|
+
|
83
|
+
def handle_operation
|
84
|
+
@_app._nodes << node
|
85
|
+
if(@_app._options[:command] == :info)
|
86
|
+
node_hash = @_node_hash_elements
|
87
|
+
node_hash[:config] = @_operation._options if @_operation.respond_to? :_options # Always add options if it exists
|
88
|
+
# Add node parameters to node hash
|
89
|
+
@_node_hash_pars.each do |par|
|
90
|
+
node_hash[par] = node.instance_variable_get("@_#{par}")
|
91
|
+
end
|
92
|
+
# Add optional node parameters to node hash
|
93
|
+
@_node_hash_optpars.each do |par|
|
94
|
+
node_hash[par] = node.instance_variable_get("@_#{par}") if node.instance_variable_defined?("@_#{par}")
|
95
|
+
end
|
96
|
+
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
|
97
|
+
|
98
|
+
# Create arc hashes from streams that feed into this node
|
99
|
+
@_arc_hash_streams.each do |stream, direction|
|
100
|
+
arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => node._name}
|
101
|
+
arc_hash[direction] = 1 if !direction.nil?
|
102
|
+
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
|
103
|
+
end
|
104
|
+
elsif(@_app._options[:command] == :execute and @_app._options[:name] == node._name)
|
105
|
+
@_operation.run_operation
|
106
|
+
end
|
107
|
+
self
|
108
|
+
end
|
109
|
+
end
|
@@ -1,26 +1,45 @@
|
|
1
1
|
class Zillabyte::Harness::Sink
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
class Node
|
5
|
+
attr_accessor :_name, :_relation, :_type, :_columns, :_scope
|
6
|
+
|
7
|
+
def initialize()
|
8
|
+
@_type = 'sink'
|
9
|
+
@_columns = []
|
10
|
+
end
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
+
def name(v)
|
13
|
+
@_name = v
|
14
|
+
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
+
def relation(v)
|
17
|
+
@_relation = v
|
18
|
+
end
|
16
19
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
20
|
+
def column(cname, ctype)
|
21
|
+
col = {cname => ctype}
|
22
|
+
@_columns << col
|
23
|
+
end
|
21
24
|
|
22
|
-
|
23
|
-
|
25
|
+
def scope(v)
|
26
|
+
@_scope = v
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(app, *args)
|
31
|
+
@_app = app
|
32
|
+
@_options = Zillabyte::Harness::Helper.get_options(args)
|
33
|
+
end
|
34
|
+
|
35
|
+
def build_node(&block)
|
36
|
+
@_node = Node.new
|
37
|
+
@_node.instance_eval(&block)
|
38
|
+
Zillabyte::Harness::Helper.check_sink(@_node, @_app._nodes)
|
39
|
+
@_node._relation ||= @_node._name
|
40
|
+
end
|
41
|
+
|
42
|
+
def run_operation
|
43
|
+
puts "Sinks cannot be run."
|
24
44
|
end
|
25
|
-
|
26
45
|
end
|
@@ -1,44 +1,73 @@
|
|
1
1
|
class Zillabyte::Harness::Source
|
2
|
-
|
3
|
-
attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_end_cycle_policy, :_begin_cycle, :_next_tuple, :_prepare
|
2
|
+
attr_accessor :_app, :_node, :_relation
|
4
3
|
|
5
|
-
|
6
|
-
|
7
|
-
@_type = 'source'
|
8
|
-
@_end_cycle_policy = :null_emit
|
9
|
-
end
|
4
|
+
class Node < Zillabyte::Harness::CommonNode
|
5
|
+
attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple, :_prepare
|
10
6
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
@
|
15
|
-
when Array
|
16
|
-
@_matches = v
|
7
|
+
def initialize()
|
8
|
+
@_name = "source_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = 'source'
|
10
|
+
@_end_cycle_policy = :null_emit
|
17
11
|
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def name(v)
|
21
|
-
@_name = v
|
22
|
-
end
|
23
12
|
|
24
|
-
|
25
|
-
|
26
|
-
|
13
|
+
def matches(v, options = {})
|
14
|
+
case v
|
15
|
+
when String
|
16
|
+
@_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
|
17
|
+
when Array
|
18
|
+
@_matches = v
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def parallelism(v)
|
23
|
+
throw "parallelism cannot be specified for sources"
|
24
|
+
end
|
25
|
+
|
26
|
+
def end_cycle_policy(v)
|
27
|
+
@_end_cycle_policy = v
|
28
|
+
end
|
27
29
|
|
28
|
-
|
29
|
-
|
30
|
-
|
30
|
+
def begin_cycle(&block)
|
31
|
+
@_begin_cycle = block
|
32
|
+
end
|
31
33
|
|
32
|
-
|
33
|
-
|
34
|
+
def next_tuple(&block)
|
35
|
+
@_next_tuple = block
|
36
|
+
end
|
37
|
+
|
38
|
+
def prepare(&block)
|
39
|
+
@_prepare = block
|
40
|
+
end
|
34
41
|
end
|
35
42
|
|
36
|
-
def
|
37
|
-
@
|
43
|
+
def initialize(app, *args)
|
44
|
+
@_app = app
|
45
|
+
@_args = args
|
38
46
|
end
|
39
|
-
|
40
|
-
def
|
41
|
-
@
|
47
|
+
|
48
|
+
def build_node(&block)
|
49
|
+
@_node = Node.new
|
50
|
+
# Are we given a block?
|
51
|
+
if block_given?
|
52
|
+
@_node.instance_eval(&block)
|
53
|
+
Zillabyte::Harness::Helper.check_name("source", @_node._name, @_app._names)
|
54
|
+
Zillabyte::Harness::Helper.check_custom_source(@_node)
|
55
|
+
if @_node._emits
|
56
|
+
Zillabyte::Harness::Helper.check_emits("source", @_node._emits, @_app._streams)
|
57
|
+
else
|
58
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
59
|
+
end
|
60
|
+
# No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
|
61
|
+
else
|
62
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
63
|
+
Zillabyte::Harness::Helper.check_source_args(@_args)
|
64
|
+
@_node.matches(@_args[0])
|
65
|
+
end
|
42
66
|
end
|
43
67
|
|
68
|
+
def run_operation
|
69
|
+
pipe_name = @_app._options[:pipe]
|
70
|
+
c = Zillabyte::Harness::SourceController.new(@_node, progress = Zillabyte::Common::Progress.new)
|
71
|
+
c.run(pipe_name)
|
72
|
+
end
|
44
73
|
end
|