zillabyte 0.1.43 → 0.1.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -384,7 +384,9 @@ class Zillabyte::Harness::Helper
384
384
  end
385
385
 
386
386
  def self.get_non_option_args(args)
387
- if args.last.is_a?(Hash)
387
+ if args.nil?
388
+ return []
389
+ elsif args.last.is_a?(Hash)
388
390
  return args[0..-2]
389
391
  else
390
392
  return args
@@ -1,31 +1,55 @@
1
1
  class Zillabyte::Harness::InjectedComponent
2
- attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
2
+ attr_accessor :_app, :_node, :_id, :_input_stream_1
3
3
 
4
- def initialize()
5
- @_name = "component_"+Zillabyte::Harness::Counter.get()
6
- @_type = "component"
7
- @_consumes = []
8
- @_output_format = :replace
9
- end
4
+ class Node
5
+ attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
10
6
 
11
- def name(v)
12
- @_name = v
13
- end
7
+ def initialize(input_stream_1)
8
+ @_name = "component_"+Zillabyte::Harness::Counter.get()
9
+ @_type = "component"
10
+ @_consumes = [input_stream_1]
11
+ @_output_format = :replace
12
+ end
14
13
 
15
- def component_id(v)
16
- @_id = v.to_s
17
- end
14
+ def name(v)
15
+ @_name = v
16
+ end
17
+
18
+ def component_id(v)
19
+ @_id = v.to_s
20
+ end
18
21
 
19
- def additional_inputs(*v)
20
- @_consumes = *v
22
+ def additional_inputs(*v)
23
+ @_consumes.push(*v)
24
+ end
25
+
26
+ def outputs(*v)
27
+ @_emits = *v
28
+ end
29
+
30
+ def output_format(v)
31
+ @_output_format = v
32
+ end
21
33
  end
22
34
 
23
- def outputs(*v)
24
- @_emits = *v
35
+ def initialize(app, *args)
36
+ @_app = app
37
+ @_input_stream_1 = args[0]
38
+ @_id = args[1]
25
39
  end
26
40
 
27
- def output_format(v)
28
- @_output_format = v
41
+ def build_node(&block)
42
+ @_node = Node.new @_input_stream_1
43
+ @_node.instance_eval(&block) if block_given?
44
+ @_node._id ||= @_id
45
+ @_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()] # We default to single-stream when called like this..
46
+
47
+ Zillabyte::Harness::Helper.check_name("component", @_node._name, @_app._names)
48
+ Zillabyte::Harness::Helper.check_call_component(@_node)
49
+ Zillabyte::Harness::Helper.check_emits("component", @_node._emits, @_app._streams)
29
50
  end
30
51
 
52
+ def run_operation
53
+ puts "Components can only be instantiated on our servers, sorry!"
54
+ end
31
55
  end
@@ -1,39 +1,67 @@
1
1
  class Zillabyte::Harness::Join
2
- attr_accessor :_name, :_type, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
3
- #:_group_by, :_emits, :_begin_group, :_aggregate, :_end_group
4
-
5
- def initialize(options = {})
6
- @_name = "join_"+Zillabyte::Harness::Counter.get()
7
- @_type = 'join'
8
- @_join_type = :inner
9
-
10
- options.symbolize_keys!
11
-
12
- emits = options[:emits]
13
- if emits
14
- if emits.is_a?(String)
15
- @_emits = [options[:emits]]
16
- else
17
- @_emits = options[:emits]
2
+ attr_accessor :_app, :_node, :_args
3
+
4
+ class Node
5
+ attr_accessor :_name, :_type, :_lhs_stream, :_rhs_stream, :_lhs_fields, :_rhs_fields, :_join_type, :_emits
6
+
7
+ def initialize(*args)
8
+ @_name = "join_"+Zillabyte::Harness::Counter.get()
9
+ @_type = 'join'
10
+ @_join_type = :inner
11
+ @_lhs_stream = args[0]
12
+ @_rhs_stream = args[1]
13
+ options = args[2]
14
+
15
+ options.symbolize_keys!
16
+
17
+ emits = options[:emits]
18
+ if emits
19
+ if emits.is_a?(String)
20
+ @_emits = [options[:emits]]
21
+ else
22
+ @_emits = options[:emits]
23
+ end
18
24
  end
19
- end
20
25
 
21
- # Figure out which fields to join on..
22
- on = options[:on]
23
- if on
24
- if on.is_a?(Array) and on.size == 2
25
- @_lhs_fields = on[0]
26
- @_rhs_fields = on[1]
27
- elsif (on.is_a?(String) or on.is_a?(Symbol))
28
- @_lhs_fields = on
29
- @_rhs_fields = on
26
+ # Figure out which fields to join on..
27
+ on = options[:on]
28
+ if on
29
+ if on.is_a?(Array) and on.size == 2
30
+ @_lhs_fields = on[0]
31
+ @_rhs_fields = on[1]
32
+ elsif (on.is_a?(String) or on.is_a?(Symbol))
33
+ @_lhs_fields = on
34
+ @_rhs_fields = on
35
+ end
30
36
  end
31
- end
32
37
 
33
- # Different kind of join types...
34
- if options[:type]
35
- @_join_type = options[:type].to_sym
38
+ # Different kind of join types...
39
+ if options[:type]
40
+ @_join_type = options[:type].to_sym
41
+ end
42
+ end
43
+ end
44
+
45
+ def initialize(app, *args)
46
+ @_app = app
47
+ @_args = args
48
+ end
49
+
50
+ def build_node
51
+ @_node = Node.new(*@_args)
52
+ Zillabyte::Harness::Helper.check_name("join", @_node._name, @_app._names)
53
+
54
+ # Build the node
55
+ if @_node._emits
56
+ Zillabyte::Harness::Helper.check_emits("join", @_node._emits, @_app._streams)
57
+ else
58
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
36
59
  end
37
-
60
+
61
+ Zillabyte::Harness::Helper.check_join(@_node)
62
+ end
63
+
64
+ def run_operation
65
+ puts "Join operations can only be instantiated on our cluster, sorry!"
38
66
  end
39
67
  end
@@ -0,0 +1,28 @@
1
+ class Zillabyte::Harness::JvmOperation
2
+ attr_accessor :_type, :_node, :_args
3
+
4
+ class Node
5
+ attr_accessor :_name, :_type, :_emits
6
+
7
+ def initialize(type)
8
+ @_name = "#{type}_"+Zillabyte::Harness::Counter.get()
9
+ @_type = type
10
+ end
11
+
12
+ end
13
+
14
+ def initialize(type, *args)
15
+ @_type = type
16
+ @_args = args
17
+ end
18
+
19
+ def build_node
20
+ @_node = Node.new @_type
21
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
22
+ @_node
23
+ end
24
+
25
+ def run_operation
26
+ puts "#{@_type} operation can only be instantiated on our cluster, sorry!"
27
+ end
28
+ end
@@ -0,0 +1,109 @@
1
+ class Zillabyte::Harness::OperationHandler
2
+ attr_accessor :_app, :_stream_class, :_operation, :_node_hash_elements, :_node_hash_pars, :_node_hash_optpars, :_arc_hash_streams
3
+
4
+ def initialize(app, stream_class)
5
+ @_app = app
6
+ @_stream_class = stream_class
7
+ @_node_hash_elements = {}
8
+ @_node_hash_pars = []
9
+ @_node_hash_optpars = []
10
+ @_arc_hash_streams = {}
11
+ end
12
+
13
+ def node
14
+ @_operation._node
15
+ end
16
+
17
+ def build_multilang_operation(type, *args, &block)
18
+ case type
19
+ when "source"
20
+ @_operation = Zillabyte::Harness::Source.new(@_app, *args)
21
+ when "each"
22
+ @_operation = Zillabyte::Harness::Each.new(@_app, *args)
23
+ when "filter"
24
+ @_operation = Zillabyte::Harness::Filter.new(@_app, *args)
25
+ when "group_by"
26
+ @_operation = Zillabyte::Harness::GroupBy.new(@_app, *args)
27
+ when "join"
28
+ @_operation = Zillabyte::Harness::Join.new(@_app, *args)
29
+ when "component"
30
+ @_operation = Zillabyte::Harness::InjectedComponent.new(@_app, *args)
31
+ when "sink"
32
+ @_operation = Zillabyte::Harness::Sink.new(@_app, *args)
33
+ when "component_source"
34
+ @_operation = Zillabyte::Harness::ComponentSource.new(@_app, *args)
35
+ when "component_sink"
36
+ @_operation = Zillabyte::Harness::ComponentSink.new(@_app, *args)
37
+ else
38
+ throw "Unknown type"
39
+ end
40
+ @_operation.build_node(&block)
41
+ self
42
+ end
43
+
44
+ def build_jvm_operation(type, *args)
45
+ @_operation = Zillabyte::Harness::JvmOperation.new(type, *args)
46
+ @_operation.build_node
47
+ self
48
+ end
49
+
50
+ def add_operation_properties_to_info(*args)
51
+ @_node_hash_pars.concat(args)
52
+ self
53
+ end
54
+
55
+ def add_optional_operation_properties_to_info(*args)
56
+ @_node_hash_optpars.concat(args)
57
+ self
58
+ end
59
+
60
+ def add_input_args_to_info_as(key, index = nil)
61
+ if !index.nil?
62
+ @_node_hash_elements[key] = @_operation._args[index]
63
+ else
64
+ @_node_hash_elements[key] = @_operation._args
65
+ end
66
+ self
67
+ end
68
+
69
+ def create_arc_info_from_stream(stream, direction = nil)
70
+ @_arc_hash_streams[stream] = direction
71
+ self
72
+ end
73
+
74
+ def get_output_streams(&block)
75
+ output_streams = []
76
+ node._emits.each do |stream|
77
+ output_streams << @_stream_class.new(stream, @_app, node._name)
78
+ end
79
+ output_streams = output_streams[0] if output_streams.size == 1
80
+ output_streams
81
+ end
82
+
83
+ def handle_operation
84
+ @_app._nodes << node
85
+ if(@_app._options[:command] == :info)
86
+ node_hash = @_node_hash_elements
87
+ node_hash[:config] = @_operation._options if @_operation.respond_to? :_options # Always add options if it exists
88
+ # Add node parameters to node hash
89
+ @_node_hash_pars.each do |par|
90
+ node_hash[par] = node.instance_variable_get("@_#{par}")
91
+ end
92
+ # Add optional node parameters to node hash
93
+ @_node_hash_optpars.each do |par|
94
+ node_hash[par] = node.instance_variable_get("@_#{par}") if node.instance_variable_defined?("@_#{par}")
95
+ end
96
+ Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
97
+
98
+ # Create arc hashes from streams that feed into this node
99
+ @_arc_hash_streams.each do |stream, direction|
100
+ arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => node._name}
101
+ arc_hash[direction] = 1 if !direction.nil?
102
+ Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
103
+ end
104
+ elsif(@_app._options[:command] == :execute and @_app._options[:name] == node._name)
105
+ @_operation.run_operation
106
+ end
107
+ self
108
+ end
109
+ end
@@ -1,26 +1,45 @@
1
1
  class Zillabyte::Harness::Sink
2
- attr_accessor :_name, :_relation, :_type, :_columns, :_scope
2
+ attr_accessor :_app, :_node, :_options
3
3
 
4
- def initialize()
5
- @_type = 'sink'
6
- @_columns = []
7
- end
4
+ class Node
5
+ attr_accessor :_name, :_relation, :_type, :_columns, :_scope
6
+
7
+ def initialize()
8
+ @_type = 'sink'
9
+ @_columns = []
10
+ end
8
11
 
9
- def name(v)
10
- @_name = v
11
- end
12
+ def name(v)
13
+ @_name = v
14
+ end
12
15
 
13
- def relation(v)
14
- @_relation = v
15
- end
16
+ def relation(v)
17
+ @_relation = v
18
+ end
16
19
 
17
- def column(cname, ctype)
18
- col = {cname => ctype}
19
- @_columns << col
20
- end
20
+ def column(cname, ctype)
21
+ col = {cname => ctype}
22
+ @_columns << col
23
+ end
21
24
 
22
- def scope(v)
23
- @_scope = v
25
+ def scope(v)
26
+ @_scope = v
27
+ end
28
+ end
29
+
30
+ def initialize(app, *args)
31
+ @_app = app
32
+ @_options = Zillabyte::Harness::Helper.get_options(args)
33
+ end
34
+
35
+ def build_node(&block)
36
+ @_node = Node.new
37
+ @_node.instance_eval(&block)
38
+ Zillabyte::Harness::Helper.check_sink(@_node, @_app._nodes)
39
+ @_node._relation ||= @_node._name
40
+ end
41
+
42
+ def run_operation
43
+ puts "Sinks cannot be run."
24
44
  end
25
-
26
45
  end
@@ -1,44 +1,73 @@
1
1
  class Zillabyte::Harness::Source
2
-
3
- attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_end_cycle_policy, :_begin_cycle, :_next_tuple, :_prepare
2
+ attr_accessor :_app, :_node, :_relation
4
3
 
5
- def initialize()
6
- @_name = "source_"+Zillabyte::Harness::Counter.get()
7
- @_type = 'source'
8
- @_end_cycle_policy = :null_emit
9
- end
4
+ class Node < Zillabyte::Harness::CommonNode
5
+ attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple, :_prepare
10
6
 
11
- def matches(v, options = {})
12
- case v
13
- when String
14
- @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
15
- when Array
16
- @_matches = v
7
+ def initialize()
8
+ @_name = "source_"+Zillabyte::Harness::Counter.get()
9
+ @_type = 'source'
10
+ @_end_cycle_policy = :null_emit
17
11
  end
18
- end
19
-
20
- def name(v)
21
- @_name = v
22
- end
23
12
 
24
- def emits(*v)
25
- @_emits = *v
26
- end
13
+ def matches(v, options = {})
14
+ case v
15
+ when String
16
+ @_relation = { :query => v, :options => options.is_a?(Hash) ? options : {} }
17
+ when Array
18
+ @_matches = v
19
+ end
20
+ end
21
+
22
+ def parallelism(v)
23
+ throw "parallelism cannot be specified for sources"
24
+ end
25
+
26
+ def end_cycle_policy(v)
27
+ @_end_cycle_policy = v
28
+ end
27
29
 
28
- def end_cycle_policy(v)
29
- @_end_cycle_policy = v
30
- end
30
+ def begin_cycle(&block)
31
+ @_begin_cycle = block
32
+ end
31
33
 
32
- def begin_cycle(&block)
33
- @_begin_cycle = block
34
+ def next_tuple(&block)
35
+ @_next_tuple = block
36
+ end
37
+
38
+ def prepare(&block)
39
+ @_prepare = block
40
+ end
34
41
  end
35
42
 
36
- def next_tuple(&block)
37
- @_next_tuple = block
43
+ def initialize(app, *args)
44
+ @_app = app
45
+ @_args = args
38
46
  end
39
-
40
- def prepare(&block)
41
- @_prepare = block
47
+
48
+ def build_node(&block)
49
+ @_node = Node.new
50
+ # Are we given a block?
51
+ if block_given?
52
+ @_node.instance_eval(&block)
53
+ Zillabyte::Harness::Helper.check_name("source", @_node._name, @_app._names)
54
+ Zillabyte::Harness::Helper.check_custom_source(@_node)
55
+ if @_node._emits
56
+ Zillabyte::Harness::Helper.check_emits("source", @_node._emits, @_app._streams)
57
+ else
58
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
59
+ end
60
+ # No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
61
+ else
62
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
63
+ Zillabyte::Harness::Helper.check_source_args(@_args)
64
+ @_node.matches(@_args[0])
65
+ end
42
66
  end
43
67
 
68
+ def run_operation
69
+ pipe_name = @_app._options[:pipe]
70
+ c = Zillabyte::Harness::SourceController.new(@_node, progress = Zillabyte::Common::Progress.new)
71
+ c.run(pipe_name)
72
+ end
44
73
  end