zillabyte 0.9.9 → 0.9.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +14 -6
- data/ruby/lib/zillabyte/harness/app.rb +0 -9
- data/ruby/lib/zillabyte/harness/component.rb +2 -0
- data/ruby/lib/zillabyte/harness/component_input.rb +2 -0
- data/ruby/lib/zillabyte/harness/component_output.rb +1 -0
- data/ruby/lib/zillabyte/harness/component_stream.rb +1 -0
- data/ruby/lib/zillabyte/harness/each.rb +6 -5
- data/ruby/lib/zillabyte/harness/filter.rb +3 -3
- data/ruby/lib/zillabyte/harness/helper.rb +6 -6
- data/ruby/lib/zillabyte/harness/injected_component.rb +3 -2
- data/ruby/lib/zillabyte/harness/jvm_operation.rb +6 -5
- data/ruby/lib/zillabyte/harness/operation_handler.rb +13 -1
- data/ruby/lib/zillabyte/harness/source.rb +2 -2
- data/ruby/lib/zillabyte/harness/stream.rb +39 -10
- data/ruby/lib/zillabyte/harness/stream_builder.rb +14 -1
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +13 -13
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
NGUxNzIxMjQyMmFmY2Y1YjNhYTg2YWJmNjcwN2U5MTIwYzM3MjJhMw==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ZTkwYTI4NDcyMTJjNDQxMmM5Y2UzODM5ZmZiYzFmNDAxZDg0NGY4Mg==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
YTM3YWE0MWNhNzI5ZDkyY2Y5YTI3ODk1M2MzZmE5MTY0OWE1ZDU5ODgwOWI0
|
10
|
+
MDhiMmY5OTNmMWJlNzQwN2RiZGM4YmE0NTEwMzdmNTBjNTRhYWM4MDA3MmYx
|
11
|
+
ZmY0ODU4NjBhOTEyNWJlY2E4MzI5ZTI3MzM4YTkxMTllYmY4YjM=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NDQzMjQ4OTk0YTVkYzVhZGU2NDk5YzM1ZWE5ZWJiN2YxMGZiOWQ4NjRkNDVl
|
14
|
+
NTNjMjIwMjhkMTdlYTdkYmY3NDQzMzQwYTczZDI3Njc0YThhNjkzYzE3MTA5
|
15
|
+
NzRkZDY0YzY0YjJiNjU5ZjNlYWI3NGUzNDUwYTNiMzMxYWRkYTY=
|
@@ -31,20 +31,11 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
|
|
31
31
|
end
|
32
32
|
|
33
33
|
|
34
|
-
def sink_to_redshift(*args, &block)
|
35
|
-
_source_common(Zillabyte::Harness::Stream, *args, &block)
|
36
|
-
end
|
37
|
-
|
38
|
-
|
39
34
|
def source_from_kinesis(*args, &block)
|
40
35
|
_source_common(Zillabyte::Harness::Stream, "kinesis_demo_source")
|
41
36
|
end
|
42
37
|
|
43
38
|
|
44
|
-
def sink_to_kinesis(*args, &block)
|
45
|
-
_source_common(Zillabyte::Harness::Stream, "kinesis_demo_sink")
|
46
|
-
end
|
47
|
-
|
48
39
|
|
49
40
|
|
50
41
|
def call_component(*args, &block)
|
@@ -54,10 +54,11 @@ class Zillabyte::Harness::Each
|
|
54
54
|
class Node < Zillabyte::Harness::CommonNode
|
55
55
|
attr_accessor :_output_format, :_execute
|
56
56
|
|
57
|
-
def initialize(
|
58
|
-
@_name =
|
57
|
+
def initialize(v, options = {})
|
58
|
+
@_name = v[0] || "each_"+Zillabyte::Harness::Counter.get()
|
59
59
|
@_type = "each"
|
60
60
|
@_output_format = :replace
|
61
|
+
self._emits = options[:emits]
|
61
62
|
end
|
62
63
|
|
63
64
|
def output_format(v)
|
@@ -71,11 +72,11 @@ class Zillabyte::Harness::Each
|
|
71
72
|
|
72
73
|
def initialize(app, *args)
|
73
74
|
@_app = app
|
74
|
-
@
|
75
|
+
@_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
|
75
76
|
end
|
76
77
|
|
77
78
|
def build_node(&block)
|
78
|
-
@_node = Node.new(
|
79
|
+
@_node = Node.new(@_args, @_options)
|
79
80
|
# Does the block take 0 arguments? If so it's not just an execute block.
|
80
81
|
if(block.arity == 0)
|
81
82
|
@_node.instance_eval(&block)
|
@@ -87,7 +88,7 @@ class Zillabyte::Harness::Each
|
|
87
88
|
end
|
88
89
|
# Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
|
89
90
|
else
|
90
|
-
@_node._emits
|
91
|
+
@_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()]
|
91
92
|
@_node._execute = block
|
92
93
|
end
|
93
94
|
|
@@ -39,7 +39,7 @@ class Zillabyte::Harness::Filter
|
|
39
39
|
class Node < Zillabyte::Harness::CommonNode
|
40
40
|
attr_accessor :_keep
|
41
41
|
|
42
|
-
def initialize()
|
42
|
+
def initialize(args, options = {})
|
43
43
|
@_name = "filter_"+Zillabyte::Harness::Counter.get()
|
44
44
|
@_type = 'filter'
|
45
45
|
end
|
@@ -51,11 +51,11 @@ class Zillabyte::Harness::Filter
|
|
51
51
|
|
52
52
|
def initialize(app, *args)
|
53
53
|
@_app = app
|
54
|
-
@
|
54
|
+
@_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
|
55
55
|
end
|
56
56
|
|
57
57
|
def build_node(&block)
|
58
|
-
@_node = Node.new
|
58
|
+
@_node = Node.new(@_args, @_options)
|
59
59
|
# Does the block take 0 arguments? If so it's not just an execute block.
|
60
60
|
if(block.arity == 0)
|
61
61
|
@_node.instance_eval(&block)
|
@@ -408,12 +408,12 @@ class Zillabyte::Harness::Helper
|
|
408
408
|
Zillabyte::Harness::Helper.print_error(msg)
|
409
409
|
end
|
410
410
|
end
|
411
|
-
if node._name == previous_node_name
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
end
|
411
|
+
# if node._name == previous_node_name
|
412
|
+
# if node._emits.size < 2
|
413
|
+
# msg = "#{ee}The preceding operation does not emit multiple streams. Please make sure that it emits one stream for the loop back and another stream for downstream operations. #{pp}"
|
414
|
+
# Zillabyte::Harness::Helper.print_error(msg)
|
415
|
+
# end
|
416
|
+
# end
|
417
417
|
end
|
418
418
|
if !found
|
419
419
|
msg = "#{ee}The specified loop-back node \"#{node_name}\" was not found in the operations preceding it. #{pp}"
|
@@ -4,11 +4,12 @@ class Zillabyte::Harness::InjectedComponent
|
|
4
4
|
class Node
|
5
5
|
attr_accessor :_name, :_type, :_id, :_consumes, :_emits, :_output_format
|
6
6
|
|
7
|
-
def initialize(input_stream_1)
|
7
|
+
def initialize(input_stream_1, options = {})
|
8
8
|
@_name = "component_"+Zillabyte::Harness::Counter.get()
|
9
9
|
@_type = "component"
|
10
10
|
@_consumes = [input_stream_1]
|
11
11
|
@_output_format = :replace
|
12
|
+
@_emits ||= options[:emits]
|
12
13
|
end
|
13
14
|
|
14
15
|
def name(v)
|
@@ -46,7 +47,7 @@ class Zillabyte::Harness::InjectedComponent
|
|
46
47
|
end
|
47
48
|
|
48
49
|
def build_node(&block)
|
49
|
-
@_node = Node.new @_input_stream_1
|
50
|
+
@_node = Node.new @_input_stream_1, @_options
|
50
51
|
@_node.instance_eval(&block) if block_given?
|
51
52
|
@_node._id ||= @_id
|
52
53
|
@_node._emits ||= ["stream_"+Zillabyte::Harness::Counter.get()] # We default to single-stream when called like this..
|
@@ -1,23 +1,24 @@
|
|
1
1
|
class Zillabyte::Harness::JvmOperation
|
2
|
-
attr_accessor :_type, :_node, :_args
|
2
|
+
attr_accessor :_type, :_node, :_args, :_options
|
3
3
|
|
4
4
|
class Node
|
5
5
|
attr_accessor :_name, :_type, :_emits
|
6
6
|
|
7
|
-
def initialize(type)
|
8
|
-
@_name = "#{type}_"+Zillabyte::Harness::Counter.get()
|
7
|
+
def initialize(type, options = {})
|
8
|
+
@_name = options[:name] || "#{type}_"+Zillabyte::Harness::Counter.get()
|
9
9
|
@_type = type
|
10
|
+
@_options = options
|
10
11
|
end
|
11
12
|
|
12
13
|
end
|
13
14
|
|
14
15
|
def initialize(type, *args)
|
15
16
|
@_type = type
|
16
|
-
@_args = args
|
17
|
+
@_args, @_options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
|
17
18
|
end
|
18
19
|
|
19
20
|
def build_node
|
20
|
-
@_node = Node.new @_type
|
21
|
+
@_node = Node.new @_type, @_options
|
21
22
|
@_node._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
22
23
|
@_node
|
23
24
|
end
|
@@ -80,7 +80,7 @@ class Zillabyte::Harness::OperationHandler
|
|
80
80
|
output_streams
|
81
81
|
end
|
82
82
|
|
83
|
-
def handle_operation
|
83
|
+
def handle_operation()
|
84
84
|
@_app._nodes << node
|
85
85
|
if(@_app._options[:command] == :info)
|
86
86
|
node_hash = @_node_hash_elements
|
@@ -106,4 +106,16 @@ class Zillabyte::Harness::OperationHandler
|
|
106
106
|
end
|
107
107
|
self
|
108
108
|
end
|
109
|
+
|
110
|
+
|
111
|
+
def self.build(stream, operaiton_name, *args)
|
112
|
+
op = Zillabyte::Harness::OperationHandler.new(stream._app, stream.class)
|
113
|
+
op.build_jvm_operation(operaiton_name, *args)
|
114
|
+
.add_operation_properties_to_info(:name, :type)
|
115
|
+
.create_arc_info_from_stream(stream)
|
116
|
+
.handle_operation
|
117
|
+
.get_output_streams
|
118
|
+
end
|
119
|
+
|
120
|
+
|
109
121
|
end
|
@@ -61,7 +61,7 @@ class Zillabyte::Harness::Source
|
|
61
61
|
class Node < Zillabyte::Harness::CommonNode
|
62
62
|
attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
|
63
63
|
|
64
|
-
def initialize(options = {})
|
64
|
+
def initialize(v, options = {})
|
65
65
|
@_name = options[:name] || "source_"+Zillabyte::Harness::Counter.get()
|
66
66
|
@_type = 'source'
|
67
67
|
@_end_cycle_policy = :null_emit
|
@@ -101,7 +101,7 @@ class Zillabyte::Harness::Source
|
|
101
101
|
end
|
102
102
|
|
103
103
|
def build_node(&block)
|
104
|
-
@_node = Node.new(@_options)
|
104
|
+
@_node = Node.new(@_args, @_options)
|
105
105
|
# Are we given a block?
|
106
106
|
if block_given?
|
107
107
|
@_node.instance_eval(&block)
|
@@ -13,7 +13,6 @@ class Zillabyte::Harness::Stream
|
|
13
13
|
@_name
|
14
14
|
end
|
15
15
|
|
16
|
-
|
17
16
|
def each(*args, &block)
|
18
17
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
19
18
|
op.build_multilang_operation("each", *args, &block)
|
@@ -89,7 +88,9 @@ class Zillabyte::Harness::Stream
|
|
89
88
|
Zillabyte::Harness::Helper.check_loop_back(self, loop_back_node, max_iterations, @_app._nodes)
|
90
89
|
arc_hash = {"name" => (@_options[:name] || self._name), "origin" => self._previous_node_name, "dest" => loop_back_node, "loop_back" => 1}
|
91
90
|
arc_hash["max_iterations"] = max_iterations if max_iterations
|
92
|
-
|
91
|
+
if(@_app._options[:command] == :info)
|
92
|
+
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
|
93
|
+
end
|
93
94
|
end
|
94
95
|
|
95
96
|
def sink(*args, &block)
|
@@ -102,6 +103,18 @@ class Zillabyte::Harness::Stream
|
|
102
103
|
end
|
103
104
|
|
104
105
|
|
106
|
+
def sink_to_redshift(*args, &block)
|
107
|
+
sink(*args, &block)
|
108
|
+
end
|
109
|
+
|
110
|
+
|
111
|
+
|
112
|
+
def sink_to_kinesis(*args, &block)
|
113
|
+
sink(*args, &block)
|
114
|
+
end
|
115
|
+
|
116
|
+
|
117
|
+
|
105
118
|
|
106
119
|
# Unique's a stream. Executed on backend
|
107
120
|
def unique(*args)
|
@@ -114,6 +127,26 @@ class Zillabyte::Harness::Stream
|
|
114
127
|
.get_output_streams
|
115
128
|
end
|
116
129
|
|
130
|
+
|
131
|
+
|
132
|
+
def route_by(*fields)
|
133
|
+
options = {}
|
134
|
+
options[:fields] = fields
|
135
|
+
Zillabyte::Harness::OperationHandler.build(self, "route_by", options)
|
136
|
+
end
|
137
|
+
|
138
|
+
def rate_limit(rate = 1, options = {})
|
139
|
+
options[:rate] = rate
|
140
|
+
Zillabyte::Harness::OperationHandler.build(self, "rate_limit", options)
|
141
|
+
end
|
142
|
+
|
143
|
+
def fetch_url(*args)
|
144
|
+
args, options = Zillabyte::Harness::Helper.get_vector_and_hashes(args)
|
145
|
+
self.call_component(:fetch_url, options)
|
146
|
+
end
|
147
|
+
|
148
|
+
|
149
|
+
|
117
150
|
# Counts fields. Executed on backend
|
118
151
|
def count(*args)
|
119
152
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
@@ -159,14 +192,10 @@ class Zillabyte::Harness::Stream
|
|
159
192
|
end
|
160
193
|
|
161
194
|
# Renames fields..
|
162
|
-
def rename(
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
.add_input_args_to_info_as(:rename, 0)
|
167
|
-
.create_arc_info_from_stream(self)
|
168
|
-
.handle_operation
|
169
|
-
.get_output_streams
|
195
|
+
def rename(rename_map = {})
|
196
|
+
options = {}
|
197
|
+
options[:rename] = rename_map
|
198
|
+
Zillabyte::Harness::OperationHandler.build(self, "rename", options)
|
170
199
|
end
|
171
200
|
|
172
201
|
end
|
@@ -11,7 +11,9 @@ class Zillabyte::Harness::StreamBuilder < Array
|
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
+
|
14
15
|
# Override these methods explicitly because they are provided by 'Array'
|
16
|
+
alias_method :super_each, :each
|
15
17
|
def each(*args, &block)
|
16
18
|
_handle_function(:each, *args, &block)
|
17
19
|
end
|
@@ -42,7 +44,18 @@ class Zillabyte::Harness::StreamBuilder < Array
|
|
42
44
|
if self.size == 0
|
43
45
|
raise "The previous operation did not emit a split stream"
|
44
46
|
else
|
45
|
-
|
47
|
+
if v.is_a?(String)
|
48
|
+
# String based
|
49
|
+
super_each do |stream|
|
50
|
+
if stream._name == v
|
51
|
+
return Zillabyte::Harness::StreamBuilder.new(stream)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
raise "Could not find a stream with name '#{v}'"
|
55
|
+
else
|
56
|
+
# Intger based
|
57
|
+
return Zillabyte::Harness::StreamBuilder.new(super)
|
58
|
+
end
|
46
59
|
end
|
47
60
|
end
|
48
61
|
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.9.
|
4
|
+
version: 0.9.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-10-
|
11
|
+
date: 2014-10-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - ! '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - ! '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: zillabyte-cli
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - ~>
|
32
32
|
- !ruby/object:Gem::Version
|
33
|
-
version: 0.9.
|
33
|
+
version: 0.9.10
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ~>
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: 0.9.
|
40
|
+
version: 0.9.10
|
41
41
|
description: The Official Zillabyte Gem
|
42
42
|
email:
|
43
43
|
- gem@zillabyte.com
|
@@ -45,9 +45,7 @@ executables: []
|
|
45
45
|
extensions: []
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
|
-
- ruby/lib/zillabyte.rb
|
49
48
|
- ruby/lib/zillabyte/common/progress.rb
|
50
|
-
- ruby/lib/zillabyte/harness.rb
|
51
49
|
- ruby/lib/zillabyte/harness/app.rb
|
52
50
|
- ruby/lib/zillabyte/harness/base.rb
|
53
51
|
- ruby/lib/zillabyte/harness/clump.rb
|
@@ -71,7 +69,9 @@ files:
|
|
71
69
|
- ruby/lib/zillabyte/harness/stream.rb
|
72
70
|
- ruby/lib/zillabyte/harness/stream_builder.rb
|
73
71
|
- ruby/lib/zillabyte/harness/tuple.rb
|
72
|
+
- ruby/lib/zillabyte/harness.rb
|
74
73
|
- ruby/lib/zillabyte/version.rb
|
74
|
+
- ruby/lib/zillabyte.rb
|
75
75
|
homepage: http://www.zillabyte.com
|
76
76
|
licenses:
|
77
77
|
- MIT
|
@@ -82,17 +82,17 @@ require_paths:
|
|
82
82
|
- ruby/lib
|
83
83
|
required_ruby_version: !ruby/object:Gem::Requirement
|
84
84
|
requirements:
|
85
|
-
- -
|
85
|
+
- - ! '>='
|
86
86
|
- !ruby/object:Gem::Version
|
87
87
|
version: '0'
|
88
88
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
89
89
|
requirements:
|
90
|
-
- -
|
90
|
+
- - ! '>='
|
91
91
|
- !ruby/object:Gem::Version
|
92
92
|
version: '0'
|
93
93
|
requirements: []
|
94
94
|
rubyforge_project:
|
95
|
-
rubygems_version: 2.
|
95
|
+
rubygems_version: 2.0.6
|
96
96
|
signing_key:
|
97
97
|
specification_version: 4
|
98
98
|
summary: The Official Zillabyte Gem
|