zillabyte 0.0.15 → 0.0.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +9 -9
- data/ruby/lib/zillabyte.rb +2 -2
- data/ruby/lib/zillabyte/harness/helper.rb +28 -5
- data/ruby/lib/zillabyte/harness/live_delegator.rb +22 -11
- data/ruby/lib/zillabyte/harness/simple_aggregate.rb +8 -6
- data/ruby/lib/zillabyte/harness/simple_function.rb +12 -9
- data/ruby/lib/zillabyte/harness/{simple_spout.rb → simple_source.rb} +26 -23
- data/ruby/lib/zillabyte/harness/sink.rb +5 -1
- data/ruby/lib/zillabyte/harness/{spout.rb → source.rb} +3 -3
- data/ruby/lib/zillabyte/harness/topology.rb +20 -24
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
OTIzYmY3MjJhM2U0M2I2OGU4ZGE3ZDk2YmI4NjhkYzM1MTY0ZjY1ZA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
7
|
-
|
6
|
+
OTkyMzYzODhjZmNmNjZiYzc2ZGI3NTVjNDFlNDljMzBjNTJhOTRiOQ==
|
7
|
+
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
OTJiZjQwMGRiYjEzMTM4YTZlODZhZTA2N2Y1YjNiMDQ4MTY3NWNkZjNkYzM0
|
10
|
+
NzRkNmVkN2M3OTFmNjc1Yzc2ZDU5YTA5ZjNlODU3YjIwNzVlNTE1NGM5OTk5
|
11
|
+
YjQxOThlNWNmY2E0YWUwM2YzZjM4Y2JhMzY3Y2IyM2VkMjM0ZTQ=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
N2IyMjlhMWVkZTdmODA4MmZhOWY4YTU0NTg3NDMwZTlhY2ZmOTMzZjAyNzU1
|
14
|
+
Y2E0ZTcxYzYyNmVjM2Q0ZDM5YjEyMmQ1Y2ZlOGJmY2NmZjk2MDRiMzQ1ODIx
|
15
|
+
OGUyOWY4MmRmODc4M2VkMTJiOWI5ZTY0MTVjYTZiYWM4MjE5MjQ=
|
data/ruby/lib/zillabyte.rb
CHANGED
@@ -12,8 +12,8 @@ module Zillabyte
|
|
12
12
|
Zillabyte::Harness::SimpleFunction.build(*args, &block)
|
13
13
|
end
|
14
14
|
|
15
|
-
def self.
|
16
|
-
Zillabyte::Harness::
|
15
|
+
def self.simple_source(*args, &block)
|
16
|
+
Zillabyte::Harness::SimpleSource.build(*args, &block)
|
17
17
|
end
|
18
18
|
|
19
19
|
def self.simple_aggregate(*args, &block)
|
@@ -49,6 +49,29 @@ class Zillabyte::Harness::Helper
|
|
49
49
|
|
50
50
|
end
|
51
51
|
|
52
|
+
def self.check_source(operation, source)
|
53
|
+
ee = "Error in \"#{operation}\": \n\t "
|
54
|
+
if(["simple_function", "simple_aggregate"].member?(operation))
|
55
|
+
pp = "\"#{operation}\" may only contain a single \"matches\" clause."
|
56
|
+
else
|
57
|
+
pp = "A \"#{operation}\" may only contain a single \"emits\" OR \"matches\" clause."
|
58
|
+
end
|
59
|
+
|
60
|
+
em = !source._emits.nil?
|
61
|
+
rm = !source._relation.nil?
|
62
|
+
mm = !source._matches.nil?
|
63
|
+
if( (em and rm) or (rm and mm) or (mm and em) )
|
64
|
+
msg = "#{ee}#{pp}"
|
65
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
66
|
+
end
|
67
|
+
if(rm or mm)
|
68
|
+
if source._prepare or source._next_batch
|
69
|
+
msg = "#{ee}If \"#{operation}\" has a \"matches\" clause, it may not have a \"prepare\" or \"next_batch\" block."
|
70
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
52
75
|
def self.check_emits(operation, emits, streams)
|
53
76
|
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
54
77
|
if(operation.include?("simple"))
|
@@ -133,7 +156,7 @@ class Zillabyte::Harness::Helper
|
|
133
156
|
|
134
157
|
consumes = h._consumes
|
135
158
|
if(!consumes)
|
136
|
-
msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\", \"
|
159
|
+
msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\", \"source\" or \"aggregate\" emitted multiple streams. #{pp}"
|
137
160
|
Zillabyte::Harness::Helper.print_error(msg)
|
138
161
|
end
|
139
162
|
if(!streams[consumes])
|
@@ -160,7 +183,7 @@ class Zillabyte::Harness::Helper
|
|
160
183
|
end
|
161
184
|
|
162
185
|
if(operation == "aggregate")
|
163
|
-
# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from
|
186
|
+
# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from source!
|
164
187
|
group_by.each do |f|
|
165
188
|
if(h._consumes)
|
166
189
|
fields = streams[h._consumes]
|
@@ -357,7 +380,7 @@ consumed by the aggregation function."
|
|
357
380
|
- Field names cannot be \"v[number]\", \"id\", \"confidence\", \"since\" or \"source\" which are reserved Zillabyte names.
|
358
381
|
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
359
382
|
- If there are multiple streams, \"consumes\" must be specified for each sink as a non-empty STRING!
|
360
|
-
* \"Consumes\" is the name of a stream emitted by an \"each\" or a \"
|
383
|
+
* \"Consumes\" is the name of a stream emitted by an \"each\" or a \"source\" which the \"sink\" should save as a table.
|
361
384
|
* The columns specified in \"sink\" must match the fields emitted by the stream."
|
362
385
|
|
363
386
|
@@_print_check_each_consumes = "\n
|
@@ -370,7 +393,7 @@ consumed by the aggregation function."
|
|
370
393
|
\t \t h.execute ...
|
371
394
|
\t end
|
372
395
|
- If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
|
373
|
-
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"
|
396
|
+
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"source\" or \"aggregate\" which the current \"each\" operates on."
|
374
397
|
|
375
398
|
@@_print_check_aggregate_consumes = "\n
|
376
399
|
\"Aggregate\" Syntax for multiple streams:
|
@@ -384,7 +407,7 @@ consumed by the aggregation function."
|
|
384
407
|
\t \t h.end_group ...
|
385
408
|
\t end
|
386
409
|
- If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
|
387
|
-
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"
|
410
|
+
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"source\" or \"aggregate\" which the current \"aggregate\" operates on."
|
388
411
|
|
389
412
|
@@meta_names=["id", "confidence", "since", "source"]
|
390
413
|
end
|
@@ -122,7 +122,7 @@ module Zillabyte
|
|
122
122
|
end
|
123
123
|
|
124
124
|
|
125
|
-
class
|
125
|
+
class Source
|
126
126
|
include Storm::Protocol
|
127
127
|
|
128
128
|
def open(conf, context); end
|
@@ -132,7 +132,7 @@ module Zillabyte
|
|
132
132
|
def ack(id); end
|
133
133
|
|
134
134
|
def run(pipe_name)
|
135
|
-
Storm::Protocol.mode = '
|
135
|
+
Storm::Protocol.mode = 'source'
|
136
136
|
Storm::Protocol.pipe_name = pipe_name
|
137
137
|
setup_pipes
|
138
138
|
open(*handshake)
|
@@ -151,7 +151,7 @@ module Zillabyte
|
|
151
151
|
rescue SignalException => e
|
152
152
|
raise
|
153
153
|
rescue Exception => e
|
154
|
-
fail 'Exception in
|
154
|
+
fail 'Exception in source: ' + e.message + ' - ' + e.backtrace.join('\n')
|
155
155
|
break
|
156
156
|
end
|
157
157
|
done
|
@@ -224,7 +224,7 @@ module Zillabyte
|
|
224
224
|
|
225
225
|
module Harness
|
226
226
|
|
227
|
-
class
|
227
|
+
class SourceController < Storm::Source
|
228
228
|
|
229
229
|
def initialize(harness, progress)
|
230
230
|
@harness = harness
|
@@ -232,11 +232,15 @@ module Zillabyte
|
|
232
232
|
end
|
233
233
|
|
234
234
|
def open(*args)
|
235
|
-
|
235
|
+
self.instance_exec(*args, &@harness._prepare) if @harness._prepare
|
236
|
+
#@harness._prepare.call(self, *args) if @harness._prepare
|
236
237
|
end
|
237
238
|
|
238
239
|
def next_batch
|
239
|
-
|
240
|
+
#@harness._next_batch.call(self)
|
241
|
+
# Executes the block @harness._next_batch in the context of the class self (i.e. exposes all methods of self to
|
242
|
+
# the block).
|
243
|
+
self.instance_eval &@harness._next_batch
|
240
244
|
end
|
241
245
|
|
242
246
|
end
|
@@ -249,11 +253,15 @@ module Zillabyte
|
|
249
253
|
end
|
250
254
|
|
251
255
|
def execute(*args)
|
252
|
-
|
256
|
+
#@harness._execute.call(self, *args)
|
257
|
+
# Executes the block @harness._next_batch in the context of the class self (i.e. exposes all methods of self to
|
258
|
+
# the block). Also allows you to pass in arguments to the block in *args.
|
259
|
+
self.instance_exec *args, &@harness._execute
|
253
260
|
end
|
254
261
|
|
255
262
|
def prepare(*args)
|
256
|
-
|
263
|
+
self.instance_exec(*args, &@harness._prepare) if @harness._prepare
|
264
|
+
#@harness._prepare.call(self, *args) if @harness._prepare
|
257
265
|
end
|
258
266
|
|
259
267
|
end
|
@@ -266,15 +274,18 @@ module Zillabyte
|
|
266
274
|
end
|
267
275
|
|
268
276
|
def begin_group(*args)
|
269
|
-
|
277
|
+
#@harness._begin_group.call(*args)
|
278
|
+
self.instance_exec *args, &@harness._begin_group
|
270
279
|
end
|
271
280
|
|
272
281
|
def aggregate(*args)
|
273
|
-
|
282
|
+
#@harness._aggregate.call(*args)
|
283
|
+
self.instance_exec *args, &@harness._aggregate
|
274
284
|
end
|
275
285
|
|
276
286
|
def end_group(*args)
|
277
|
-
|
287
|
+
#@harness._end_group.call(self, *args)
|
288
|
+
self.instance_exec *args, &@harness._end_group
|
278
289
|
end
|
279
290
|
|
280
291
|
end
|
@@ -3,9 +3,10 @@ require 'optparse'
|
|
3
3
|
class Zillabyte::Harness::SimpleAggregate
|
4
4
|
attr_accessor :_nodes, :_relation, :_matches, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group, :_name, :_info_file, :_options
|
5
5
|
|
6
|
-
def self.build()
|
6
|
+
def self.build(*args, &block)
|
7
7
|
h = Zillabyte::Harness::SimpleAggregate.new()
|
8
|
-
yield(h)
|
8
|
+
#yield(h)
|
9
|
+
h.instance_eval(&block)
|
9
10
|
h._name = h._name
|
10
11
|
Zillabyte::Harness::Helper.check_name("simple_aggregate", h._name, {})
|
11
12
|
Zillabyte::Harness::Helper.check_emits("simple_aggregate", h._emits, {})
|
@@ -20,7 +21,7 @@ class Zillabyte::Harness::SimpleAggregate
|
|
20
21
|
Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
|
21
22
|
end
|
22
23
|
|
23
|
-
h.
|
24
|
+
h.build_source()
|
24
25
|
fn = h.build_aggregate(generic_emits)
|
25
26
|
h.build_sink()
|
26
27
|
|
@@ -80,10 +81,11 @@ class Zillabyte::Harness::SimpleAggregate
|
|
80
81
|
generic_emits
|
81
82
|
end
|
82
83
|
|
83
|
-
def
|
84
|
-
h = Zillabyte::Harness::
|
84
|
+
def build_source()
|
85
|
+
h = Zillabyte::Harness::Source.new(false)
|
85
86
|
h._matches = @_matches if @_matches
|
86
87
|
h._relation = @_relation if @_relation
|
88
|
+
Zillabyte::Harness::Helper.check_source("simple_aggregate", h)
|
87
89
|
@_nodes << h
|
88
90
|
if(@_options[:command] == :info)
|
89
91
|
info_hash = {"name" => h._name, "type" => h._type}
|
@@ -130,7 +132,7 @@ class Zillabyte::Harness::SimpleAggregate
|
|
130
132
|
end
|
131
133
|
@_nodes << h
|
132
134
|
if(@_options[:command] == :info)
|
133
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
|
135
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
|
134
136
|
if(h._consumes)
|
135
137
|
info_hash["consumes"] = h._consumes
|
136
138
|
end
|
@@ -3,9 +3,10 @@ require 'optparse'
|
|
3
3
|
class Zillabyte::Harness::SimpleFunction
|
4
4
|
attr_accessor :_nodes, :_relation, :_matches, :_emits, :_prepare, :_execute, :_name, :_info_file, :_options
|
5
5
|
|
6
|
-
def self.build()
|
6
|
+
def self.build(*args, &block)
|
7
7
|
h = Zillabyte::Harness::SimpleFunction.new()
|
8
|
-
|
8
|
+
h.instance_eval(&block)
|
9
|
+
#yield(h)
|
9
10
|
h._name = h._name
|
10
11
|
Zillabyte::Harness::Helper.check_name("simple_function", h._name, {})
|
11
12
|
Zillabyte::Harness::Helper.check_emits("simple_function", h._emits, {})
|
@@ -20,11 +21,11 @@ class Zillabyte::Harness::SimpleFunction
|
|
20
21
|
Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
|
21
22
|
end
|
22
23
|
|
23
|
-
h.
|
24
|
+
h.build_source()
|
24
25
|
fn = h.build_each(generic_emits)
|
25
26
|
h.build_sink()
|
26
27
|
|
27
|
-
if(h._options[:command] == :execute and h._options[:name] ==
|
28
|
+
if(h._options[:command] == :execute and h._options[:name] == 'each')
|
28
29
|
pipe_name = h._options[:pipe]
|
29
30
|
c = Zillabyte::Harness::EachController.new(fn, progress = Zillabyte::Common::Progress.new)
|
30
31
|
c.run(pipe_name)
|
@@ -72,10 +73,11 @@ class Zillabyte::Harness::SimpleFunction
|
|
72
73
|
generic_emits
|
73
74
|
end
|
74
75
|
|
75
|
-
def
|
76
|
-
h = Zillabyte::Harness::
|
76
|
+
def build_source()
|
77
|
+
h = Zillabyte::Harness::Source.new(false)
|
77
78
|
h._matches = @_matches if @_matches
|
78
79
|
h._relation = @_relation if @_relation
|
80
|
+
Zillabyte::Harness::Helper.check_source("simple_function", h)
|
79
81
|
@_nodes << h
|
80
82
|
if(@_options[:command] == :info)
|
81
83
|
info_hash = {"name" => h._name, "type" => h._type}
|
@@ -90,7 +92,7 @@ class Zillabyte::Harness::SimpleFunction
|
|
90
92
|
|
91
93
|
def build_each(generic_emits)
|
92
94
|
h = Zillabyte::Harness::Each.new()
|
93
|
-
h._name =
|
95
|
+
h._name = 'each'
|
94
96
|
h._emits = generic_emits
|
95
97
|
h._prepare = @_prepare
|
96
98
|
h._execute = @_execute
|
@@ -107,7 +109,8 @@ class Zillabyte::Harness::SimpleFunction
|
|
107
109
|
n_sinks = @_emits.length
|
108
110
|
@_emits.each do |emit|
|
109
111
|
h = Zillabyte::Harness::Sink.new()
|
110
|
-
h._name =
|
112
|
+
h._name = "sink"
|
113
|
+
h._relation = emit[0]
|
111
114
|
columns = emit[1]
|
112
115
|
columns.each do |col|
|
113
116
|
col.each do |cname, ctype|
|
@@ -119,7 +122,7 @@ class Zillabyte::Harness::SimpleFunction
|
|
119
122
|
end
|
120
123
|
@_nodes << h
|
121
124
|
if(@_options[:command] == :info)
|
122
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
|
125
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
|
123
126
|
if(h._consumes)
|
124
127
|
info_hash["consumes"] = h._consumes
|
125
128
|
end
|
@@ -1,38 +1,40 @@
|
|
1
1
|
require 'optparse'
|
2
2
|
|
3
|
-
class Zillabyte::Harness::
|
3
|
+
class Zillabyte::Harness::SimpleSource
|
4
4
|
attr_accessor :_nodes, :_sink, :_name
|
5
5
|
|
6
|
-
|
7
|
-
def self.build()
|
6
|
+
def self.build(*args, &block)
|
8
7
|
|
9
8
|
# Parse the options...
|
10
9
|
options = Zillabyte::Harness::Helper.opt_parser()
|
11
|
-
flow = Zillabyte::Harness::
|
10
|
+
flow = Zillabyte::Harness::SimpleSource.new()
|
12
11
|
flow._nodes = []
|
13
12
|
flow._name = options[:name]
|
14
13
|
|
15
14
|
# Get the user logic...
|
16
|
-
yield(flow)
|
15
|
+
#yield(flow)
|
16
|
+
flow.instance_eval(&block)
|
17
17
|
if(options[:command] == :info)
|
18
18
|
_info_file = File.open(options[:file], "w+")
|
19
19
|
info_hash = {"language" => "ruby", "name" => flow._name}
|
20
20
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, _info_file)
|
21
21
|
end
|
22
22
|
|
23
|
-
# Get the
|
24
|
-
|
25
|
-
yield(
|
26
|
-
|
27
|
-
|
23
|
+
# Get the source-specific logic...
|
24
|
+
source = Zillabyte::Harness::Source.new(false)
|
25
|
+
#yield(source)
|
26
|
+
source.instance_eval(&block)
|
27
|
+
source._name = "source"
|
28
|
+
Zillabyte::Harness::Helper.check_source("simple_source", source)
|
29
|
+
flow._nodes << source
|
28
30
|
if(options[:command] == :info)
|
29
|
-
info_hash = {"name" =>
|
30
|
-
if(
|
31
|
-
info_hash["emits"] =
|
32
|
-
elsif(
|
33
|
-
info_hash["relation"] =
|
34
|
-
elsif(
|
35
|
-
info_hash["matches"] =
|
31
|
+
info_hash = {"name" => source._name, "type" => source._type}
|
32
|
+
if(source._emits)
|
33
|
+
info_hash["emits"] = source._emits
|
34
|
+
elsif(source._relation)
|
35
|
+
info_hash["relation"] = source._relation
|
36
|
+
elsif(source._matches)
|
37
|
+
info_hash["matches"] = source._matches
|
36
38
|
end
|
37
39
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, _info_file)
|
38
40
|
end
|
@@ -40,14 +42,14 @@ class Zillabyte::Harness::SimpleSpout
|
|
40
42
|
# Add the sink...
|
41
43
|
flow._nodes << flow._sink
|
42
44
|
if(options[:command] == :info)
|
43
|
-
info_hash = {"name" => flow._sink._name, "type" => flow._sink._type, "columns" => flow._sink._columns}
|
45
|
+
info_hash = {"name" => flow._sink._name, "type" => flow._sink._type, "columns" => flow._sink._columns, "relation" => flow._sink._relation || flow._sink._name}
|
44
46
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, _info_file)
|
45
47
|
end
|
46
48
|
|
47
49
|
# Execute...
|
48
|
-
if(options[:command] == :execute and options[:name] == "
|
50
|
+
if(options[:command] == :execute and options[:name] == "source")
|
49
51
|
pipe_name = options[:pipe]
|
50
|
-
c = Zillabyte::Harness::
|
52
|
+
c = Zillabyte::Harness::SourceController.new(source, progress = Zillabyte::Common::Progress.new)
|
51
53
|
c.run(pipe_name)
|
52
54
|
end
|
53
55
|
|
@@ -58,16 +60,17 @@ class Zillabyte::Harness::SimpleSpout
|
|
58
60
|
|
59
61
|
|
60
62
|
def name(v)
|
61
|
-
Zillabyte::Harness::Helper.check_name("
|
63
|
+
Zillabyte::Harness::Helper.check_name("simple_source", v, {})
|
62
64
|
@_name = v
|
63
65
|
end
|
64
66
|
|
65
67
|
|
66
68
|
def emits(v)
|
67
69
|
# Construct the sink...
|
68
|
-
Zillabyte::Harness::Helper.check_emits("
|
70
|
+
Zillabyte::Harness::Helper.check_emits("simple_source", v, {})
|
69
71
|
@_sink = Zillabyte::Harness::Sink.new()
|
70
|
-
@_sink.
|
72
|
+
@_sink._name = "sink"
|
73
|
+
@_sink._relation = v[0][0]
|
71
74
|
columns = v[0][1]
|
72
75
|
columns.each do |col|
|
73
76
|
col.each do |cname, ctype|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::Sink
|
2
|
-
attr_accessor :_name, :_type, :_columns, :_consumes
|
2
|
+
attr_accessor :_name, :_relation, :_type, :_columns, :_consumes
|
3
3
|
|
4
4
|
def initialize()
|
5
5
|
@_type = 'sink'
|
@@ -15,6 +15,10 @@ class Zillabyte::Harness::Sink
|
|
15
15
|
@_name = v
|
16
16
|
end
|
17
17
|
|
18
|
+
def relation(v)
|
19
|
+
@_relation = v
|
20
|
+
end
|
21
|
+
|
18
22
|
def column(cname, ctype)
|
19
23
|
col = {cname => ctype}
|
20
24
|
@_columns << col
|
@@ -1,9 +1,9 @@
|
|
1
|
-
class Zillabyte::Harness::
|
1
|
+
class Zillabyte::Harness::Source
|
2
2
|
attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_prepare, :_next_batch, :_flow_flag
|
3
3
|
|
4
4
|
def initialize(flow_flag)
|
5
|
-
@_name = "
|
6
|
-
@_type = '
|
5
|
+
@_name = "source_"+Zillabyte::Harness::Counter.get()
|
6
|
+
@_type = 'source'
|
7
7
|
@_flow_flag = flow_flag
|
8
8
|
end
|
9
9
|
|
@@ -20,10 +20,12 @@ class Zillabyte::Harness::Topology
|
|
20
20
|
h
|
21
21
|
end
|
22
22
|
|
23
|
-
def
|
24
|
-
h = Zillabyte::Harness::
|
25
|
-
yield(h)
|
26
|
-
|
23
|
+
def source(&block)
|
24
|
+
h = Zillabyte::Harness::Source.new(true)
|
25
|
+
#yield(h)
|
26
|
+
h.instance_eval(&block)
|
27
|
+
Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
|
28
|
+
Zillabyte::Harness::Helper.check_source("source", h)
|
27
29
|
@_nodes << h
|
28
30
|
if(@_options[:command] == :info)
|
29
31
|
info_hash = {"name" => h._name, "type" => h._type}
|
@@ -31,30 +33,22 @@ class Zillabyte::Harness::Topology
|
|
31
33
|
info_hash["relation"] = h._relation
|
32
34
|
elsif(h._matches)
|
33
35
|
info_hash["matches"] = h._matches
|
36
|
+
elsif(h._emits)
|
37
|
+
@_branched = Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams) || @_branched
|
38
|
+
info_hash["emits"] = h._emits
|
34
39
|
end
|
35
40
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def spout
|
40
|
-
h = Zillabyte::Harness::Spout.new(true)
|
41
|
-
yield(h)
|
42
|
-
Zillabyte::Harness::Helper.check_name("spout", h._name, @_names)
|
43
|
-
@_branched = Zillabyte::Harness::Helper.check_emits("spout", h._emits, @_streams) || @_branched
|
44
|
-
@_nodes << h
|
45
|
-
if(@_options[:command] == :info)
|
46
|
-
info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
|
47
|
-
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
|
48
41
|
elsif(@_options[:command] == :execute and @_options[:name] == h._name)
|
49
42
|
pipe_name = @_options[:pipe]
|
50
|
-
c = Zillabyte::Harness::
|
43
|
+
c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
|
51
44
|
c.run(pipe_name)
|
52
45
|
end
|
53
46
|
end
|
54
47
|
|
55
|
-
def each
|
48
|
+
def each(&block)
|
56
49
|
h = Zillabyte::Harness::Each.new()
|
57
|
-
yield(h)
|
50
|
+
#yield(h)
|
51
|
+
h.instance_eval(&block)
|
58
52
|
Zillabyte::Harness::Helper.check_name("each", h._name, @_names)
|
59
53
|
if(@_branched)
|
60
54
|
Zillabyte::Harness::Helper.check_consumes(h, @_streams)
|
@@ -78,9 +72,10 @@ class Zillabyte::Harness::Topology
|
|
78
72
|
end
|
79
73
|
end
|
80
74
|
|
81
|
-
def aggregate
|
75
|
+
def aggregate(&block)
|
82
76
|
h = Zillabyte::Harness::Aggregate.new()
|
83
|
-
yield(h)
|
77
|
+
#yield(h)
|
78
|
+
h.instance_eval(&block)
|
84
79
|
Zillabyte::Harness::Helper.check_name("aggregate", h._name, @_names)
|
85
80
|
if(@_branched)
|
86
81
|
Zillabyte::Harness::Helper.check_consumes(h, @_streams)
|
@@ -107,9 +102,10 @@ class Zillabyte::Harness::Topology
|
|
107
102
|
end
|
108
103
|
end
|
109
104
|
|
110
|
-
def sink
|
105
|
+
def sink(&block)
|
111
106
|
h = Zillabyte::Harness::Sink.new()
|
112
|
-
yield(h)
|
107
|
+
#yield(h)
|
108
|
+
h.instance_eval(&block)
|
113
109
|
Zillabyte::Harness::Helper.check_sink(h, @_nodes)
|
114
110
|
if(@_branched)
|
115
111
|
Zillabyte::Harness::Helper.check_consumes(h, @_streams)
|
@@ -120,7 +116,7 @@ class Zillabyte::Harness::Topology
|
|
120
116
|
end
|
121
117
|
@_nodes << h
|
122
118
|
if(@_options[:command] == :info)
|
123
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
|
119
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
|
124
120
|
if(h._consumes)
|
125
121
|
info_hash["consumes"] = h._consumes
|
126
122
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-04-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -68,9 +68,9 @@ files:
|
|
68
68
|
- ruby/lib/zillabyte/harness/live_delegator.rb
|
69
69
|
- ruby/lib/zillabyte/harness/simple_aggregate.rb
|
70
70
|
- ruby/lib/zillabyte/harness/simple_function.rb
|
71
|
-
- ruby/lib/zillabyte/harness/
|
71
|
+
- ruby/lib/zillabyte/harness/simple_source.rb
|
72
72
|
- ruby/lib/zillabyte/harness/sink.rb
|
73
|
-
- ruby/lib/zillabyte/harness/
|
73
|
+
- ruby/lib/zillabyte/harness/source.rb
|
74
74
|
- ruby/lib/zillabyte/harness/topology.rb
|
75
75
|
- ruby/lib/zillabyte/harness/tuple.rb
|
76
76
|
- ruby/lib/zillabyte/harness.rb
|
@@ -97,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
97
97
|
version: '0'
|
98
98
|
requirements: []
|
99
99
|
rubyforge_project:
|
100
|
-
rubygems_version: 2.
|
100
|
+
rubygems_version: 2.0.7
|
101
101
|
signing_key:
|
102
102
|
specification_version: 4
|
103
103
|
summary: The Official Zillabyte Gem
|