zillabyte 0.1.43 → 0.1.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c4a67906d45c018f5312141a05b16a199e8b7e00
4
- data.tar.gz: 653312b49f906d7d04a96170257f38885d051ea3
3
+ metadata.gz: 52c3a818d68e3b81d1e7e60a2265278c5bc0cd7b
4
+ data.tar.gz: 01481ba634bd7f736c9257a54eb4e8a93c2530b0
5
5
  SHA512:
6
- metadata.gz: 42530197ceae964a91870da58a0562ef82e2051e661727ca78e7c812ac728ee0206a1e01654261e8fab81c7804678d8ad6b9f7895df542bcadf4e590977cc99b
7
- data.tar.gz: e4d1b3f9bdf620fe1b8f8f4e2f500348623feb56f20082ad036eeb4f5943d520ef5a3433821102678c4defc2f8189f9b9a1041aeaadbf7e5d8f201206cbdb222
6
+ metadata.gz: f2c233cb1332221b640a94e66a4bd080cf3feae7acd4d916d5b77aa49f3503744c0ef78eb00dd198f421e4e7fcfadf3a032d0a9452b8d275b96adc086492928d
7
+ data.tar.gz: 4291a6a52ae5e47e137fd55ff6fa66f2e7056d3b0dab4d91a2a117490788ad01e8bef37bfa2ee6f7e31d5c6ee5de32d4ca4b2d2d5e830d8d3fc6db3b3ba5c1fa
@@ -16,47 +16,11 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
16
16
  end
17
17
 
18
18
  def source(*args, &block)
19
- h = Zillabyte::Harness::Source.new()
20
- # Are we given a block?
21
- if block_given?
22
- h.instance_eval(&block)
23
- Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
24
- Zillabyte::Harness::Helper.check_custom_source(h)
25
- if h._emits
26
- Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams)
27
- else
28
- h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
29
- end
30
- # No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
31
- else
32
- h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
33
- Zillabyte::Harness::Helper.check_source_args(args)
34
- h.matches(args[0])
35
- end
36
- @_nodes << h
37
- if(@_options[:command] == :info)
38
- node_hash = {"name" => h._name, "type" => h._type}
39
- if(h._relation)
40
- node_hash["relation"] = h._relation
41
- elsif(h._matches)
42
- node_hash["matches"] = h._matches
43
- else
44
- node_hash["end_cycle_policy"] = h._end_cycle_policy
45
- end
46
-
47
- Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_info_file)
48
- elsif(@_options[:command] == :execute and @_options[:name] == h._name)
49
- pipe_name = @_options[:pipe]
50
- c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
51
- c.run(pipe_name)
52
- end
53
-
54
- output_streams = []
55
- h._emits.each do |stream|
56
- output_streams << Zillabyte::Harness::Stream.new(stream, self, h._name)
57
- end
58
- output_streams = output_streams[0] if output_streams.size == 1
59
- output_streams
60
- end
61
-
19
+ op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::Stream)
20
+ op.build_multilang_operation("source", *args, &block)
21
+ .add_operation_properties_to_info(:name, :type)
22
+ .add_optional_operation_properties_to_info(:relation, :matches, :end_cycle_policy)
23
+ .handle_operation
24
+ .get_output_streams
25
+ end
62
26
  end
@@ -0,0 +1,16 @@
1
+ class Zillabyte::Harness::CommonNode
2
+ attr_accessor :_name, :_type, :_emits, :_parallelism
3
+
4
+ def name(v)
5
+ @_name = v
6
+ end
7
+
8
+ def emits(*v)
9
+ @_emits = *v
10
+ end
11
+
12
+ def parallelism(v)
13
+ @_parallelism = v
14
+ end
15
+
16
+ end
@@ -15,19 +15,10 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
15
15
  end
16
16
 
17
17
  def inputs(*args, &block)
18
- h = Zillabyte::Harness::ComponentSource.new
19
- h.instance_eval(&block)
20
- h._name ||= "comp_input.#{Zillabyte::Harness::Counter.get()}"
21
- Zillabyte::Harness::Helper.check_name("inputs", h._name, @_names)
22
- Zillabyte::Harness::Helper.check_component_source(h)
23
-
24
- if(self._options[:command] == :info)
25
- node_hash = {"name" => h._name, "type" => h._type, "fields" => h._fields}
26
- Zillabyte::Harness::Helper.write_node_to_file(node_hash, self._info_file)
27
- end
28
-
29
- # For components, each "inputs" statement only defines a single stream
30
- output_stream = Zillabyte::Harness::ComponentStream.new(h._emits.first, self, h._name)
31
- output_stream
18
+ op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::ComponentStream)
19
+ op.build_multilang_operation("component_source", *args, &block)
20
+ .add_operation_properties_to_info(:name, :type, :fields)
21
+ .handle_operation
22
+ .get_output_streams
32
23
  end
33
24
  end
@@ -1,26 +1,46 @@
1
1
  class Zillabyte::Harness::ComponentSink
2
- attr_accessor :_name, :_relation, :_type, :_columns, :_scope
2
+ attr_accessor :_app, :_node
3
3
 
4
- def initialize()
5
- @_type = 'sink'
6
- @_columns = []
7
- end
4
+ class Node
5
+ attr_accessor :_name, :_relation, :_type, :_columns, :_scope
6
+
7
+ def initialize()
8
+ @_type = 'sink'
9
+ @_columns = []
10
+ end
8
11
 
9
- def name(v)
10
- @_name = v
11
- end
12
+ def name(v)
13
+ @_name = v
14
+ end
12
15
 
13
- def relation(v)
14
- @_relation = v
15
- end
16
+ def relation(v)
17
+ @_relation = v
18
+ end
16
19
 
17
- def field(cname, ctype)
18
- col = {cname => ctype}
19
- @_columns << col
20
- end
20
+ def field(cname, ctype)
21
+ col = {cname => ctype}
22
+ @_columns << col
23
+ end
21
24
 
22
- def scope(v)
23
- @_scope = v
25
+ def scope(v)
26
+ @_scope = v
27
+ end
28
+ end
29
+
30
+ def initialize(app, *args)
31
+ @_app = app
32
+ end
33
+
34
+ def build_node(&block)
35
+ @_node = Node.new
36
+ @_node.instance_eval(&block)
37
+ @_node._name ||= "comp_output.#{Zillabyte::Harness::Counter.get()}"
38
+ Zillabyte::Harness::Helper.check_name("outputs", @_node._name, @_app._names)
39
+ Zillabyte::Harness::Helper.check_component_sink(@_node)
40
+ @_node._relation ||= @_node._name
41
+ end
42
+
43
+ def run_operation
44
+ puts "Component output is not runnable locally."
24
45
  end
25
-
26
46
  end
@@ -1,17 +1,37 @@
1
1
  class Zillabyte::Harness::ComponentSource
2
- attr_accessor :_name, :_type, :_emits, :_fields
2
+ attr_accessor :_app, :_node
3
3
 
4
- def initialize()
5
- @_type = "source"
6
- @_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
7
- @_fields = []
4
+ class Node
5
+ attr_accessor :_name, :_type, :_emits, :_fields
6
+
7
+ def initialize()
8
+ @_type = "source"
9
+ @_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
10
+ @_fields = []
11
+ end
12
+
13
+ def name(v)
14
+ @_name = v
15
+ end
16
+
17
+ def field(name, type)
18
+ @_fields << {name => type}
19
+ end
20
+ end
21
+
22
+ def initialize(app, *args)
23
+ @_app = app
8
24
  end
9
25
 
10
- def name(v)
11
- @_name = v
26
+ def build_node(&block)
27
+ @_node = Node.new
28
+ @_node.instance_eval(&block)
29
+ @_node._name ||= "comp_input.#{Zillabyte::Harness::Counter.get()}"
30
+ Zillabyte::Harness::Helper.check_name("inputs", @_node._name, @_app._names)
31
+ Zillabyte::Harness::Helper.check_component_source(@_node)
12
32
  end
13
33
 
14
- def field(name, type)
15
- @_fields << {name => type}
34
+ def run_operation
35
+ puts "Component inputs cannot be run!"
16
36
  end
17
37
  end
@@ -3,19 +3,12 @@ require 'zillabyte/harness/stream'
3
3
  class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
4
4
 
5
5
  def outputs(*args, &block)
6
- h = Zillabyte::Harness::ComponentSink.new
7
- h.instance_eval(&block)
8
- h._name ||= "comp_output.#{Zillabyte::Harness::Counter.get()}"
9
- Zillabyte::Harness::Helper.check_name("outputs", h._name, @_app._names)
10
- Zillabyte::Harness::Helper.check_component_sink(h)
11
-
12
- if(@_app._options[:command] == :info)
13
- node_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "scope" => h._scope}
14
- Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._info_file)
15
- arc_hash = {"name" => @_name, "origin" => @_previous_node_name, "dest" => h._name}
16
- Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._info_file)
17
- end
6
+ op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
7
+ op.build_multilang_operation("component_sink", *args, &block)
8
+ .add_operation_properties_to_info(:name, :type, :columns, :scope)
9
+ .add_optional_operation_properties_to_info(:relation)
10
+ .create_arc_info_from_stream(self)
11
+ .handle_operation
18
12
  end
19
13
 
20
-
21
14
  end
@@ -1,34 +1,56 @@
1
1
  class Zillabyte::Harness::Each
2
- attr_accessor :_name, :_type, :_emits, :_output_format, :_parallelism, :_prepare, :_execute
2
+ attr_accessor :_app, :_node, :_options
3
3
 
4
- def initialize()
5
- @_name = "each_"+Zillabyte::Harness::Counter.get()
6
- @_type = "each"
7
- @_output_format = :replace
8
- end
9
-
10
- def name(v)
11
- @_name = v
12
- end
4
+ class Node < Zillabyte::Harness::CommonNode
5
+ attr_accessor :_output_format, :_prepare, :_execute
6
+
7
+ def initialize()
8
+ @_name = "each_"+Zillabyte::Harness::Counter.get()
9
+ @_type = "each"
10
+ @_output_format = :replace
11
+ end
13
12
 
14
- def emits(*v)
15
- @_emits = *v
16
- end
13
+ def output_format(v)
14
+ @_output_format = v
15
+ end
17
16
 
18
- def output_format(v)
19
- @_output_format = v
20
- end
17
+ def prepare(&block)
18
+ @_prepare = block
19
+ end
21
20
 
22
- def parallelism(v)
23
- @_parallelism = v
21
+ def execute(&block)
22
+ @_execute = block
23
+ end
24
24
  end
25
25
 
26
- def prepare(&block)
27
- @_prepare = block
26
+ def initialize(app, *args)
27
+ @_app = app
28
+ @_options = Zillabyte::Harness::Helper.get_options(args)
28
29
  end
29
30
 
30
- def execute(&block)
31
- @_execute = block
31
+ def build_node(&block)
32
+ @_node = Node.new
33
+ # Does the block take 0 arguments? If so it's not just an execute block.
34
+ if(block.arity == 0)
35
+ @_node.instance_eval(&block)
36
+ Zillabyte::Harness::Helper.check_name("each", @_node._name, @_app._names)
37
+ if @_node._emits
38
+ Zillabyte::Harness::Helper.check_emits("each", @_node._emits, @_app._streams)
39
+ else
40
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
41
+ end
42
+ # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
43
+ else
44
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
45
+ @_node._execute = block
46
+ end
47
+
48
+ Zillabyte::Harness::Helper.check_each(@_node)
32
49
  end
33
50
 
51
+ def run_operation
52
+ pipe_name = @_app._options[:pipe]
53
+ c = Zillabyte::Harness::EachController.new(@_node, progress = Zillabyte::Common::Progress.new)
54
+ c.run(pipe_name)
55
+ end
34
56
  end
@@ -1,29 +1,52 @@
1
1
  class Zillabyte::Harness::Filter
2
- attr_accessor :_name, :_type, :_emits, :_parallelism, :_prepare, :_keep
2
+ attr_accessor :_app, :_node, :_options
3
3
 
4
- def initialize()
5
- @_name = "filter_"+Zillabyte::Harness::Counter.get()
6
- @_type = 'filter'
7
- end
4
+ class Node < Zillabyte::Harness::CommonNode
5
+ attr_accessor :_prepare, :_keep
6
+
7
+ def initialize()
8
+ @_name = "filter_"+Zillabyte::Harness::Counter.get()
9
+ @_type = 'filter'
10
+ end
8
11
 
9
- def name(v)
10
- @_name = v
11
- end
12
+ def prepare(&block)
13
+ @_prepare = block
14
+ end
12
15
 
13
- def emits(*v)
14
- @_emits = *v
16
+ def keep(&block)
17
+ @_keep = block
18
+ end
15
19
  end
16
20
 
17
- def parallelism(v)
18
- @_parallelism = v
21
+ def initialize(app, *args)
22
+ @_app = app
23
+ @_options = Zillabyte::Harness::Helper.get_options(args)
19
24
  end
20
-
21
- def prepare(&block)
22
- @_prepare = block
25
+
26
+ def build_node(&block)
27
+ @_node = Node.new
28
+ # Does the block take 0 arguments? If so it's not just an execute block.
29
+ if(block.arity == 0)
30
+ @_node.instance_eval(&block)
31
+ Zillabyte::Harness::Helper.check_name("filter", @_node._name, @_app._names)
32
+ if @_node._emits
33
+ Zillabyte::Harness::Helper.check_emits("filter", @_node._emits, @_app._streams)
34
+ else
35
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
36
+ end
37
+ # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
38
+ else
39
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
40
+ @_node._keep = block
41
+ end
42
+
43
+ Zillabyte::Harness::Helper.check_filter(@_node)
23
44
  end
24
45
 
25
- def keep(&block)
26
- @_keep = block
46
+ def run_operation
47
+ pipe_name = @_app._options[:pipe]
48
+ c = Zillabyte::Harness::FilterController.new(@_node, progress = Zillabyte::Common::Progress.new)
49
+ c.run(pipe_name)
27
50
  end
28
51
 
29
52
  end
@@ -1,34 +1,54 @@
1
1
  class Zillabyte::Harness::GroupBy
2
- attr_accessor :_name, :_type, :_group_by, :_emits, :_parallelism, :_begin_group, :_aggregate, :_end_group
2
+ attr_accessor :_app, :_node, :_fields, :_options
3
3
 
4
- def initialize(*v)
5
- @_name = "group_by_"+Zillabyte::Harness::Counter.get()
6
- @_type = 'group_by'
7
- @_group_by = *v
8
- end
4
+ class Node < Zillabyte::Harness::CommonNode
5
+ attr_accessor :_group_by, :_begin_group, :_aggregate, :_end_group
6
+
7
+ def initialize(v)
8
+ @_name = "group_by_"+Zillabyte::Harness::Counter.get()
9
+ @_type = 'group_by'
10
+ @_group_by = v
11
+ end
9
12
 
10
- def name(v)
11
- @_name = v
12
- end
13
+ def begin_group(&block)
14
+ @_begin_group = block
15
+ end
13
16
 
14
- def emits(*v)
15
- @_emits = *v
16
- end
17
+ def aggregate(&block)
18
+ @_aggregate = block
19
+ end
17
20
 
18
- def parallelism(v)
19
- @_parallelism = v
20
- end
21
-
22
- def begin_group(&block)
23
- @_begin_group = block
21
+ def end_group(&block)
22
+ @_end_group = block
23
+ end
24
24
  end
25
25
 
26
- def aggregate(&block)
27
- @_aggregate = block
26
+ def initialize(app, *args)
27
+ @_app = app
28
+ @_fields = Zillabyte::Harness::Helper.get_non_option_args(args)
29
+ @_options = Zillabyte::Harness::Helper.get_options(args)
28
30
  end
29
31
 
30
- def end_group(&block)
31
- @_end_group = block
32
+ def build_node(&block)
33
+ @_node = Node.new(@_fields)
34
+
35
+ # Yield
36
+ @_node.instance_eval(&block)
37
+ Zillabyte::Harness::Helper.check_name("group_by", @_node._name, @_app._names)
38
+
39
+ # Multiple emits?
40
+ if @_node._emits
41
+ Zillabyte::Harness::Helper.check_emits("group_by", @_node._emits, @_app._streams)
42
+ else
43
+ @_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
44
+ end
45
+
46
+ Zillabyte::Harness::Helper.check_group_by(@_node)
32
47
  end
33
48
 
49
+ def run_operation
50
+ pipe_name = @_app._options[:pipe]
51
+ c = Zillabyte::Harness::GroupByController.new(@_node, Zillabyte::Common::Progress.new)
52
+ c.run(pipe_name)
53
+ end
34
54
  end