zillabyte 0.1.43 → 0.1.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ruby/lib/zillabyte/harness/app.rb +7 -43
- data/ruby/lib/zillabyte/harness/common_node.rb +16 -0
- data/ruby/lib/zillabyte/harness/component.rb +5 -14
- data/ruby/lib/zillabyte/harness/component_sink.rb +38 -18
- data/ruby/lib/zillabyte/harness/component_source.rb +29 -9
- data/ruby/lib/zillabyte/harness/component_stream.rb +6 -13
- data/ruby/lib/zillabyte/harness/each.rb +44 -22
- data/ruby/lib/zillabyte/harness/filter.rb +40 -17
- data/ruby/lib/zillabyte/harness/group_by.rb +42 -22
- data/ruby/lib/zillabyte/harness/helper.rb +3 -1
- data/ruby/lib/zillabyte/harness/injected_component.rb +43 -19
- data/ruby/lib/zillabyte/harness/join.rb +59 -31
- data/ruby/lib/zillabyte/harness/jvm_operation.rb +28 -0
- data/ruby/lib/zillabyte/harness/operation_handler.rb +109 -0
- data/ruby/lib/zillabyte/harness/sink.rb +37 -18
- data/ruby/lib/zillabyte/harness/source.rb +60 -31
- data/ruby/lib/zillabyte/harness/stream.rb +92 -431
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +7 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 52c3a818d68e3b81d1e7e60a2265278c5bc0cd7b
|
4
|
+
data.tar.gz: 01481ba634bd7f736c9257a54eb4e8a93c2530b0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f2c233cb1332221b640a94e66a4bd080cf3feae7acd4d916d5b77aa49f3503744c0ef78eb00dd198f421e4e7fcfadf3a032d0a9452b8d275b96adc086492928d
|
7
|
+
data.tar.gz: 4291a6a52ae5e47e137fd55ff6fa66f2e7056d3b0dab4d91a2a117490788ad01e8bef37bfa2ee6f7e31d5c6ee5de32d4ca4b2d2d5e830d8d3fc6db3b3ba5c1fa
|
@@ -16,47 +16,11 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def source(*args, &block)
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams)
|
27
|
-
else
|
28
|
-
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
29
|
-
end
|
30
|
-
# No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
|
31
|
-
else
|
32
|
-
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
33
|
-
Zillabyte::Harness::Helper.check_source_args(args)
|
34
|
-
h.matches(args[0])
|
35
|
-
end
|
36
|
-
@_nodes << h
|
37
|
-
if(@_options[:command] == :info)
|
38
|
-
node_hash = {"name" => h._name, "type" => h._type}
|
39
|
-
if(h._relation)
|
40
|
-
node_hash["relation"] = h._relation
|
41
|
-
elsif(h._matches)
|
42
|
-
node_hash["matches"] = h._matches
|
43
|
-
else
|
44
|
-
node_hash["end_cycle_policy"] = h._end_cycle_policy
|
45
|
-
end
|
46
|
-
|
47
|
-
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_info_file)
|
48
|
-
elsif(@_options[:command] == :execute and @_options[:name] == h._name)
|
49
|
-
pipe_name = @_options[:pipe]
|
50
|
-
c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
|
51
|
-
c.run(pipe_name)
|
52
|
-
end
|
53
|
-
|
54
|
-
output_streams = []
|
55
|
-
h._emits.each do |stream|
|
56
|
-
output_streams << Zillabyte::Harness::Stream.new(stream, self, h._name)
|
57
|
-
end
|
58
|
-
output_streams = output_streams[0] if output_streams.size == 1
|
59
|
-
output_streams
|
60
|
-
end
|
61
|
-
|
19
|
+
op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::Stream)
|
20
|
+
op.build_multilang_operation("source", *args, &block)
|
21
|
+
.add_operation_properties_to_info(:name, :type)
|
22
|
+
.add_optional_operation_properties_to_info(:relation, :matches, :end_cycle_policy)
|
23
|
+
.handle_operation
|
24
|
+
.get_output_streams
|
25
|
+
end
|
62
26
|
end
|
@@ -15,19 +15,10 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
|
|
15
15
|
end
|
16
16
|
|
17
17
|
def inputs(*args, &block)
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
if(self._options[:command] == :info)
|
25
|
-
node_hash = {"name" => h._name, "type" => h._type, "fields" => h._fields}
|
26
|
-
Zillabyte::Harness::Helper.write_node_to_file(node_hash, self._info_file)
|
27
|
-
end
|
28
|
-
|
29
|
-
# For components, each "inputs" statement only defines a single stream
|
30
|
-
output_stream = Zillabyte::Harness::ComponentStream.new(h._emits.first, self, h._name)
|
31
|
-
output_stream
|
18
|
+
op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::ComponentStream)
|
19
|
+
op.build_multilang_operation("component_source", *args, &block)
|
20
|
+
.add_operation_properties_to_info(:name, :type, :fields)
|
21
|
+
.handle_operation
|
22
|
+
.get_output_streams
|
32
23
|
end
|
33
24
|
end
|
@@ -1,26 +1,46 @@
|
|
1
1
|
class Zillabyte::Harness::ComponentSink
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
class Node
|
5
|
+
attr_accessor :_name, :_relation, :_type, :_columns, :_scope
|
6
|
+
|
7
|
+
def initialize()
|
8
|
+
@_type = 'sink'
|
9
|
+
@_columns = []
|
10
|
+
end
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
+
def name(v)
|
13
|
+
@_name = v
|
14
|
+
end
|
12
15
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
+
def relation(v)
|
17
|
+
@_relation = v
|
18
|
+
end
|
16
19
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
20
|
+
def field(cname, ctype)
|
21
|
+
col = {cname => ctype}
|
22
|
+
@_columns << col
|
23
|
+
end
|
21
24
|
|
22
|
-
|
23
|
-
|
25
|
+
def scope(v)
|
26
|
+
@_scope = v
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def initialize(app, *args)
|
31
|
+
@_app = app
|
32
|
+
end
|
33
|
+
|
34
|
+
def build_node(&block)
|
35
|
+
@_node = Node.new
|
36
|
+
@_node.instance_eval(&block)
|
37
|
+
@_node._name ||= "comp_output.#{Zillabyte::Harness::Counter.get()}"
|
38
|
+
Zillabyte::Harness::Helper.check_name("outputs", @_node._name, @_app._names)
|
39
|
+
Zillabyte::Harness::Helper.check_component_sink(@_node)
|
40
|
+
@_node._relation ||= @_node._name
|
41
|
+
end
|
42
|
+
|
43
|
+
def run_operation
|
44
|
+
puts "Component output is not runnable locally."
|
24
45
|
end
|
25
|
-
|
26
46
|
end
|
@@ -1,17 +1,37 @@
|
|
1
1
|
class Zillabyte::Harness::ComponentSource
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
class Node
|
5
|
+
attr_accessor :_name, :_type, :_emits, :_fields
|
6
|
+
|
7
|
+
def initialize()
|
8
|
+
@_type = "source"
|
9
|
+
@_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
10
|
+
@_fields = []
|
11
|
+
end
|
12
|
+
|
13
|
+
def name(v)
|
14
|
+
@_name = v
|
15
|
+
end
|
16
|
+
|
17
|
+
def field(name, type)
|
18
|
+
@_fields << {name => type}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def initialize(app, *args)
|
23
|
+
@_app = app
|
8
24
|
end
|
9
25
|
|
10
|
-
def
|
11
|
-
@
|
26
|
+
def build_node(&block)
|
27
|
+
@_node = Node.new
|
28
|
+
@_node.instance_eval(&block)
|
29
|
+
@_node._name ||= "comp_input.#{Zillabyte::Harness::Counter.get()}"
|
30
|
+
Zillabyte::Harness::Helper.check_name("inputs", @_node._name, @_app._names)
|
31
|
+
Zillabyte::Harness::Helper.check_component_source(@_node)
|
12
32
|
end
|
13
33
|
|
14
|
-
def
|
15
|
-
|
34
|
+
def run_operation
|
35
|
+
puts "Component inputs cannot be run!"
|
16
36
|
end
|
17
37
|
end
|
@@ -3,19 +3,12 @@ require 'zillabyte/harness/stream'
|
|
3
3
|
class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
|
4
4
|
|
5
5
|
def outputs(*args, &block)
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
if(@_app._options[:command] == :info)
|
13
|
-
node_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "scope" => h._scope}
|
14
|
-
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
|
15
|
-
arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
|
16
|
-
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
|
17
|
-
end
|
6
|
+
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
7
|
+
op.build_multilang_operation("component_sink", *args, &block)
|
8
|
+
.add_operation_properties_to_info(:name, :type, :columns, :scope)
|
9
|
+
.add_optional_operation_properties_to_info(:relation)
|
10
|
+
.create_arc_info_from_stream(self)
|
11
|
+
.handle_operation
|
18
12
|
end
|
19
13
|
|
20
|
-
|
21
14
|
end
|
@@ -1,34 +1,56 @@
|
|
1
1
|
class Zillabyte::Harness::Each
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
end
|
4
|
+
class Node < Zillabyte::Harness::CommonNode
|
5
|
+
attr_accessor :_output_format, :_prepare, :_execute
|
6
|
+
|
7
|
+
def initialize()
|
8
|
+
@_name = "each_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = "each"
|
10
|
+
@_output_format = :replace
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
def output_format(v)
|
14
|
+
@_output_format = v
|
15
|
+
end
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
17
|
+
def prepare(&block)
|
18
|
+
@_prepare = block
|
19
|
+
end
|
21
20
|
|
22
|
-
|
23
|
-
|
21
|
+
def execute(&block)
|
22
|
+
@_execute = block
|
23
|
+
end
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
@
|
26
|
+
def initialize(app, *args)
|
27
|
+
@_app = app
|
28
|
+
@_options = Zillabyte::Harness::Helper.get_options(args)
|
28
29
|
end
|
29
30
|
|
30
|
-
def
|
31
|
-
@
|
31
|
+
def build_node(&block)
|
32
|
+
@_node = Node.new
|
33
|
+
# Does the block take 0 arguments? If so it's not just an execute block.
|
34
|
+
if(block.arity == 0)
|
35
|
+
@_node.instance_eval(&block)
|
36
|
+
Zillabyte::Harness::Helper.check_name("each", @_node._name, @_app._names)
|
37
|
+
if @_node._emits
|
38
|
+
Zillabyte::Harness::Helper.check_emits("each", @_node._emits, @_app._streams)
|
39
|
+
else
|
40
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
41
|
+
end
|
42
|
+
# Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
|
43
|
+
else
|
44
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
45
|
+
@_node._execute = block
|
46
|
+
end
|
47
|
+
|
48
|
+
Zillabyte::Harness::Helper.check_each(@_node)
|
32
49
|
end
|
33
50
|
|
51
|
+
def run_operation
|
52
|
+
pipe_name = @_app._options[:pipe]
|
53
|
+
c = Zillabyte::Harness::EachController.new(@_node, progress = Zillabyte::Common::Progress.new)
|
54
|
+
c.run(pipe_name)
|
55
|
+
end
|
34
56
|
end
|
@@ -1,29 +1,52 @@
|
|
1
1
|
class Zillabyte::Harness::Filter
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
class Node < Zillabyte::Harness::CommonNode
|
5
|
+
attr_accessor :_prepare, :_keep
|
6
|
+
|
7
|
+
def initialize()
|
8
|
+
@_name = "filter_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = 'filter'
|
10
|
+
end
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
+
def prepare(&block)
|
13
|
+
@_prepare = block
|
14
|
+
end
|
12
15
|
|
13
|
-
|
14
|
-
|
16
|
+
def keep(&block)
|
17
|
+
@_keep = block
|
18
|
+
end
|
15
19
|
end
|
16
20
|
|
17
|
-
def
|
18
|
-
@
|
21
|
+
def initialize(app, *args)
|
22
|
+
@_app = app
|
23
|
+
@_options = Zillabyte::Harness::Helper.get_options(args)
|
19
24
|
end
|
20
|
-
|
21
|
-
def
|
22
|
-
@
|
25
|
+
|
26
|
+
def build_node(&block)
|
27
|
+
@_node = Node.new
|
28
|
+
# Does the block take 0 arguments? If so it's not just an execute block.
|
29
|
+
if(block.arity == 0)
|
30
|
+
@_node.instance_eval(&block)
|
31
|
+
Zillabyte::Harness::Helper.check_name("filter", @_node._name, @_app._names)
|
32
|
+
if @_node._emits
|
33
|
+
Zillabyte::Harness::Helper.check_emits("filter", @_node._emits, @_app._streams)
|
34
|
+
else
|
35
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
36
|
+
end
|
37
|
+
# Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
|
38
|
+
else
|
39
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
40
|
+
@_node._keep = block
|
41
|
+
end
|
42
|
+
|
43
|
+
Zillabyte::Harness::Helper.check_filter(@_node)
|
23
44
|
end
|
24
45
|
|
25
|
-
def
|
26
|
-
|
46
|
+
def run_operation
|
47
|
+
pipe_name = @_app._options[:pipe]
|
48
|
+
c = Zillabyte::Harness::FilterController.new(@_node, progress = Zillabyte::Common::Progress.new)
|
49
|
+
c.run(pipe_name)
|
27
50
|
end
|
28
51
|
|
29
52
|
end
|
@@ -1,34 +1,54 @@
|
|
1
1
|
class Zillabyte::Harness::GroupBy
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_app, :_node, :_fields, :_options
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
4
|
+
class Node < Zillabyte::Harness::CommonNode
|
5
|
+
attr_accessor :_group_by, :_begin_group, :_aggregate, :_end_group
|
6
|
+
|
7
|
+
def initialize(v)
|
8
|
+
@_name = "group_by_"+Zillabyte::Harness::Counter.get()
|
9
|
+
@_type = 'group_by'
|
10
|
+
@_group_by = v
|
11
|
+
end
|
9
12
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
+
def begin_group(&block)
|
14
|
+
@_begin_group = block
|
15
|
+
end
|
13
16
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
+
def aggregate(&block)
|
18
|
+
@_aggregate = block
|
19
|
+
end
|
17
20
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
def begin_group(&block)
|
23
|
-
@_begin_group = block
|
21
|
+
def end_group(&block)
|
22
|
+
@_end_group = block
|
23
|
+
end
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
27
|
-
@
|
26
|
+
def initialize(app, *args)
|
27
|
+
@_app = app
|
28
|
+
@_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
|
29
|
+
@_options = Zillabyte::Harness::Helper.get_options(args)
|
28
30
|
end
|
29
31
|
|
30
|
-
def
|
31
|
-
@
|
32
|
+
def build_node(&block)
|
33
|
+
@_node = Node.new(@_fields)
|
34
|
+
|
35
|
+
# Yield
|
36
|
+
@_node.instance_eval(&block)
|
37
|
+
Zillabyte::Harness::Helper.check_name("group_by", @_node._name, @_app._names)
|
38
|
+
|
39
|
+
# Multiple emits?
|
40
|
+
if @_node._emits
|
41
|
+
Zillabyte::Harness::Helper.check_emits("group_by", @_node._emits, @_app._streams)
|
42
|
+
else
|
43
|
+
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
44
|
+
end
|
45
|
+
|
46
|
+
Zillabyte::Harness::Helper.check_group_by(@_node)
|
32
47
|
end
|
33
48
|
|
49
|
+
def run_operation
|
50
|
+
pipe_name = @_app._options[:pipe]
|
51
|
+
c = Zillabyte::Harness::GroupByController.new(@_node, Zillabyte::Common::Progress.new)
|
52
|
+
c.run(pipe_name)
|
53
|
+
end
|
34
54
|
end
|