zillabyte 0.0.15 → 0.0.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +9 -9
- data/ruby/lib/zillabyte.rb +2 -2
- data/ruby/lib/zillabyte/harness/helper.rb +28 -5
- data/ruby/lib/zillabyte/harness/live_delegator.rb +22 -11
- data/ruby/lib/zillabyte/harness/simple_aggregate.rb +8 -6
- data/ruby/lib/zillabyte/harness/simple_function.rb +12 -9
- data/ruby/lib/zillabyte/harness/{simple_spout.rb → simple_source.rb} +26 -23
- data/ruby/lib/zillabyte/harness/sink.rb +5 -1
- data/ruby/lib/zillabyte/harness/{spout.rb → source.rb} +3 -3
- data/ruby/lib/zillabyte/harness/topology.rb +20 -24
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
OTIzYmY3MjJhM2U0M2I2OGU4ZGE3ZDk2YmI4NjhkYzM1MTY0ZjY1ZA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
7
|
-
|
6
|
+
OTkyMzYzODhjZmNmNjZiYzc2ZGI3NTVjNDFlNDljMzBjNTJhOTRiOQ==
|
7
|
+
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
OTJiZjQwMGRiYjEzMTM4YTZlODZhZTA2N2Y1YjNiMDQ4MTY3NWNkZjNkYzM0
|
10
|
+
NzRkNmVkN2M3OTFmNjc1Yzc2ZDU5YTA5ZjNlODU3YjIwNzVlNTE1NGM5OTk5
|
11
|
+
YjQxOThlNWNmY2E0YWUwM2YzZjM4Y2JhMzY3Y2IyM2VkMjM0ZTQ=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
N2IyMjlhMWVkZTdmODA4MmZhOWY4YTU0NTg3NDMwZTlhY2ZmOTMzZjAyNzU1
|
14
|
+
Y2E0ZTcxYzYyNmVjM2Q0ZDM5YjEyMmQ1Y2ZlOGJmY2NmZjk2MDRiMzQ1ODIx
|
15
|
+
OGUyOWY4MmRmODc4M2VkMTJiOWI5ZTY0MTVjYTZiYWM4MjE5MjQ=
|
data/ruby/lib/zillabyte.rb
CHANGED
@@ -12,8 +12,8 @@ module Zillabyte
|
|
12
12
|
Zillabyte::Harness::SimpleFunction.build(*args, &block)
|
13
13
|
end
|
14
14
|
|
15
|
-
def self.
|
16
|
-
Zillabyte::Harness::
|
15
|
+
def self.simple_source(*args, &block)
|
16
|
+
Zillabyte::Harness::SimpleSource.build(*args, &block)
|
17
17
|
end
|
18
18
|
|
19
19
|
def self.simple_aggregate(*args, &block)
|
@@ -49,6 +49,29 @@ class Zillabyte::Harness::Helper
|
|
49
49
|
|
50
50
|
end
|
51
51
|
|
52
|
+
def self.check_source(operation, source)
|
53
|
+
ee = "Error in \"#{operation}\": \n\t "
|
54
|
+
if(["simple_function", "simple_aggregate"].member?(operation))
|
55
|
+
pp = "\"#{operation}\" may only contain a single \"matches\" clause."
|
56
|
+
else
|
57
|
+
pp = "A \"#{operation}\" may only contain a single \"emits\" OR \"matches\" clause."
|
58
|
+
end
|
59
|
+
|
60
|
+
em = !source._emits.nil?
|
61
|
+
rm = !source._relation.nil?
|
62
|
+
mm = !source._matches.nil?
|
63
|
+
if( (em and rm) or (rm and mm) or (mm and em) )
|
64
|
+
msg = "#{ee}#{pp}"
|
65
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
66
|
+
end
|
67
|
+
if(rm or mm)
|
68
|
+
if source._prepare or source._next_batch
|
69
|
+
msg = "#{ee}If \"#{operation}\" has a \"matches\" clause, it may not have a \"prepare\" or \"next_batch\" block."
|
70
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
52
75
|
def self.check_emits(operation, emits, streams)
|
53
76
|
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
54
77
|
if(operation.include?("simple"))
|
@@ -133,7 +156,7 @@ class Zillabyte::Harness::Helper
|
|
133
156
|
|
134
157
|
consumes = h._consumes
|
135
158
|
if(!consumes)
|
136
|
-
msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\", \"
|
159
|
+
msg = "#{ee}\"Consumes\" must be specified since a preceding \"each\", \"source\" or \"aggregate\" emitted multiple streams. #{pp}"
|
137
160
|
Zillabyte::Harness::Helper.print_error(msg)
|
138
161
|
end
|
139
162
|
if(!streams[consumes])
|
@@ -160,7 +183,7 @@ class Zillabyte::Harness::Helper
|
|
160
183
|
end
|
161
184
|
|
162
185
|
if(operation == "aggregate")
|
163
|
-
# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from
|
186
|
+
# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from source!
|
164
187
|
group_by.each do |f|
|
165
188
|
if(h._consumes)
|
166
189
|
fields = streams[h._consumes]
|
@@ -357,7 +380,7 @@ consumed by the aggregation function."
|
|
357
380
|
- Field names cannot be \"v[number]\", \"id\", \"confidence\", \"since\" or \"source\" which are reserved Zillabyte names.
|
358
381
|
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
359
382
|
- If there are multiple streams, \"consumes\" must be specified for each sink as a non-empty STRING!
|
360
|
-
* \"Consumes\" is the name of a stream emitted by an \"each\" or a \"
|
383
|
+
* \"Consumes\" is the name of a stream emitted by an \"each\" or a \"source\" which the \"sink\" should save as a table.
|
361
384
|
* The columns specified in \"sink\" must match the fields emitted by the stream."
|
362
385
|
|
363
386
|
@@_print_check_each_consumes = "\n
|
@@ -370,7 +393,7 @@ consumed by the aggregation function."
|
|
370
393
|
\t \t h.execute ...
|
371
394
|
\t end
|
372
395
|
- If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
|
373
|
-
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"
|
396
|
+
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"source\" or \"aggregate\" which the current \"each\" operates on."
|
374
397
|
|
375
398
|
@@_print_check_aggregate_consumes = "\n
|
376
399
|
\"Aggregate\" Syntax for multiple streams:
|
@@ -384,7 +407,7 @@ consumed by the aggregation function."
|
|
384
407
|
\t \t h.end_group ...
|
385
408
|
\t end
|
386
409
|
- If there are multiple streams, \"consumes\" must be specified as a non-empty STRING!
|
387
|
-
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"
|
410
|
+
* \"Consumes\" is the name of a stream emitted by a preceding \"each\", \"source\" or \"aggregate\" which the current \"aggregate\" operates on."
|
388
411
|
|
389
412
|
@@meta_names=["id", "confidence", "since", "source"]
|
390
413
|
end
|
@@ -122,7 +122,7 @@ module Zillabyte
|
|
122
122
|
end
|
123
123
|
|
124
124
|
|
125
|
-
class
|
125
|
+
class Source
|
126
126
|
include Storm::Protocol
|
127
127
|
|
128
128
|
def open(conf, context); end
|
@@ -132,7 +132,7 @@ module Zillabyte
|
|
132
132
|
def ack(id); end
|
133
133
|
|
134
134
|
def run(pipe_name)
|
135
|
-
Storm::Protocol.mode = '
|
135
|
+
Storm::Protocol.mode = 'source'
|
136
136
|
Storm::Protocol.pipe_name = pipe_name
|
137
137
|
setup_pipes
|
138
138
|
open(*handshake)
|
@@ -151,7 +151,7 @@ module Zillabyte
|
|
151
151
|
rescue SignalException => e
|
152
152
|
raise
|
153
153
|
rescue Exception => e
|
154
|
-
fail 'Exception in
|
154
|
+
fail 'Exception in source: ' + e.message + ' - ' + e.backtrace.join('\n')
|
155
155
|
break
|
156
156
|
end
|
157
157
|
done
|
@@ -224,7 +224,7 @@ module Zillabyte
|
|
224
224
|
|
225
225
|
module Harness
|
226
226
|
|
227
|
-
class
|
227
|
+
class SourceController < Storm::Source
|
228
228
|
|
229
229
|
def initialize(harness, progress)
|
230
230
|
@harness = harness
|
@@ -232,11 +232,15 @@ module Zillabyte
|
|
232
232
|
end
|
233
233
|
|
234
234
|
def open(*args)
|
235
|
-
|
235
|
+
self.instance_exec(*args, &@harness._prepare) if @harness._prepare
|
236
|
+
#@harness._prepare.call(self, *args) if @harness._prepare
|
236
237
|
end
|
237
238
|
|
238
239
|
def next_batch
|
239
|
-
|
240
|
+
#@harness._next_batch.call(self)
|
241
|
+
# Executes the block @harness._next_batch in the context of the class self (i.e. exposes all methods of self to
|
242
|
+
# the block).
|
243
|
+
self.instance_eval &@harness._next_batch
|
240
244
|
end
|
241
245
|
|
242
246
|
end
|
@@ -249,11 +253,15 @@ module Zillabyte
|
|
249
253
|
end
|
250
254
|
|
251
255
|
def execute(*args)
|
252
|
-
|
256
|
+
#@harness._execute.call(self, *args)
|
257
|
+
# Executes the block @harness._next_batch in the context of the class self (i.e. exposes all methods of self to
|
258
|
+
# the block). Also allows you to pass in arguments to the block in *args.
|
259
|
+
self.instance_exec *args, &@harness._execute
|
253
260
|
end
|
254
261
|
|
255
262
|
def prepare(*args)
|
256
|
-
|
263
|
+
self.instance_exec(*args, &@harness._prepare) if @harness._prepare
|
264
|
+
#@harness._prepare.call(self, *args) if @harness._prepare
|
257
265
|
end
|
258
266
|
|
259
267
|
end
|
@@ -266,15 +274,18 @@ module Zillabyte
|
|
266
274
|
end
|
267
275
|
|
268
276
|
def begin_group(*args)
|
269
|
-
|
277
|
+
#@harness._begin_group.call(*args)
|
278
|
+
self.instance_exec *args, &@harness._begin_group
|
270
279
|
end
|
271
280
|
|
272
281
|
def aggregate(*args)
|
273
|
-
|
282
|
+
#@harness._aggregate.call(*args)
|
283
|
+
self.instance_exec *args, &@harness._aggregate
|
274
284
|
end
|
275
285
|
|
276
286
|
def end_group(*args)
|
277
|
-
|
287
|
+
#@harness._end_group.call(self, *args)
|
288
|
+
self.instance_exec *args, &@harness._end_group
|
278
289
|
end
|
279
290
|
|
280
291
|
end
|
@@ -3,9 +3,10 @@ require 'optparse'
|
|
3
3
|
class Zillabyte::Harness::SimpleAggregate
|
4
4
|
attr_accessor :_nodes, :_relation, :_matches, :_group_by, :_emits, :_begin_group, :_aggregate, :_end_group, :_name, :_info_file, :_options
|
5
5
|
|
6
|
-
def self.build()
|
6
|
+
def self.build(*args, &block)
|
7
7
|
h = Zillabyte::Harness::SimpleAggregate.new()
|
8
|
-
yield(h)
|
8
|
+
#yield(h)
|
9
|
+
h.instance_eval(&block)
|
9
10
|
h._name = h._name
|
10
11
|
Zillabyte::Harness::Helper.check_name("simple_aggregate", h._name, {})
|
11
12
|
Zillabyte::Harness::Helper.check_emits("simple_aggregate", h._emits, {})
|
@@ -20,7 +21,7 @@ class Zillabyte::Harness::SimpleAggregate
|
|
20
21
|
Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
|
21
22
|
end
|
22
23
|
|
23
|
-
h.
|
24
|
+
h.build_source()
|
24
25
|
fn = h.build_aggregate(generic_emits)
|
25
26
|
h.build_sink()
|
26
27
|
|
@@ -80,10 +81,11 @@ class Zillabyte::Harness::SimpleAggregate
|
|
80
81
|
generic_emits
|
81
82
|
end
|
82
83
|
|
83
|
-
def
|
84
|
-
h = Zillabyte::Harness::
|
84
|
+
def build_source()
|
85
|
+
h = Zillabyte::Harness::Source.new(false)
|
85
86
|
h._matches = @_matches if @_matches
|
86
87
|
h._relation = @_relation if @_relation
|
88
|
+
Zillabyte::Harness::Helper.check_source("simple_aggregate", h)
|
87
89
|
@_nodes << h
|
88
90
|
if(@_options[:command] == :info)
|
89
91
|
info_hash = {"name" => h._name, "type" => h._type}
|
@@ -130,7 +132,7 @@ class Zillabyte::Harness::SimpleAggregate
|
|
130
132
|
end
|
131
133
|
@_nodes << h
|
132
134
|
if(@_options[:command] == :info)
|
133
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
|
135
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
|
134
136
|
if(h._consumes)
|
135
137
|
info_hash["consumes"] = h._consumes
|
136
138
|
end
|
@@ -3,9 +3,10 @@ require 'optparse'
|
|
3
3
|
class Zillabyte::Harness::SimpleFunction
|
4
4
|
attr_accessor :_nodes, :_relation, :_matches, :_emits, :_prepare, :_execute, :_name, :_info_file, :_options
|
5
5
|
|
6
|
-
def self.build()
|
6
|
+
def self.build(*args, &block)
|
7
7
|
h = Zillabyte::Harness::SimpleFunction.new()
|
8
|
-
|
8
|
+
h.instance_eval(&block)
|
9
|
+
#yield(h)
|
9
10
|
h._name = h._name
|
10
11
|
Zillabyte::Harness::Helper.check_name("simple_function", h._name, {})
|
11
12
|
Zillabyte::Harness::Helper.check_emits("simple_function", h._emits, {})
|
@@ -20,11 +21,11 @@ class Zillabyte::Harness::SimpleFunction
|
|
20
21
|
Zillabyte::Harness::Helper.write_hash_to_file(hash, h._info_file)
|
21
22
|
end
|
22
23
|
|
23
|
-
h.
|
24
|
+
h.build_source()
|
24
25
|
fn = h.build_each(generic_emits)
|
25
26
|
h.build_sink()
|
26
27
|
|
27
|
-
if(h._options[:command] == :execute and h._options[:name] ==
|
28
|
+
if(h._options[:command] == :execute and h._options[:name] == 'each')
|
28
29
|
pipe_name = h._options[:pipe]
|
29
30
|
c = Zillabyte::Harness::EachController.new(fn, progress = Zillabyte::Common::Progress.new)
|
30
31
|
c.run(pipe_name)
|
@@ -72,10 +73,11 @@ class Zillabyte::Harness::SimpleFunction
|
|
72
73
|
generic_emits
|
73
74
|
end
|
74
75
|
|
75
|
-
def
|
76
|
-
h = Zillabyte::Harness::
|
76
|
+
def build_source()
|
77
|
+
h = Zillabyte::Harness::Source.new(false)
|
77
78
|
h._matches = @_matches if @_matches
|
78
79
|
h._relation = @_relation if @_relation
|
80
|
+
Zillabyte::Harness::Helper.check_source("simple_function", h)
|
79
81
|
@_nodes << h
|
80
82
|
if(@_options[:command] == :info)
|
81
83
|
info_hash = {"name" => h._name, "type" => h._type}
|
@@ -90,7 +92,7 @@ class Zillabyte::Harness::SimpleFunction
|
|
90
92
|
|
91
93
|
def build_each(generic_emits)
|
92
94
|
h = Zillabyte::Harness::Each.new()
|
93
|
-
h._name =
|
95
|
+
h._name = 'each'
|
94
96
|
h._emits = generic_emits
|
95
97
|
h._prepare = @_prepare
|
96
98
|
h._execute = @_execute
|
@@ -107,7 +109,8 @@ class Zillabyte::Harness::SimpleFunction
|
|
107
109
|
n_sinks = @_emits.length
|
108
110
|
@_emits.each do |emit|
|
109
111
|
h = Zillabyte::Harness::Sink.new()
|
110
|
-
h._name =
|
112
|
+
h._name = "sink"
|
113
|
+
h._relation = emit[0]
|
111
114
|
columns = emit[1]
|
112
115
|
columns.each do |col|
|
113
116
|
col.each do |cname, ctype|
|
@@ -119,7 +122,7 @@ class Zillabyte::Harness::SimpleFunction
|
|
119
122
|
end
|
120
123
|
@_nodes << h
|
121
124
|
if(@_options[:command] == :info)
|
122
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
|
125
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
|
123
126
|
if(h._consumes)
|
124
127
|
info_hash["consumes"] = h._consumes
|
125
128
|
end
|
@@ -1,38 +1,40 @@
|
|
1
1
|
require 'optparse'
|
2
2
|
|
3
|
-
class Zillabyte::Harness::
|
3
|
+
class Zillabyte::Harness::SimpleSource
|
4
4
|
attr_accessor :_nodes, :_sink, :_name
|
5
5
|
|
6
|
-
|
7
|
-
def self.build()
|
6
|
+
def self.build(*args, &block)
|
8
7
|
|
9
8
|
# Parse the options...
|
10
9
|
options = Zillabyte::Harness::Helper.opt_parser()
|
11
|
-
flow = Zillabyte::Harness::
|
10
|
+
flow = Zillabyte::Harness::SimpleSource.new()
|
12
11
|
flow._nodes = []
|
13
12
|
flow._name = options[:name]
|
14
13
|
|
15
14
|
# Get the user logic...
|
16
|
-
yield(flow)
|
15
|
+
#yield(flow)
|
16
|
+
flow.instance_eval(&block)
|
17
17
|
if(options[:command] == :info)
|
18
18
|
_info_file = File.open(options[:file], "w+")
|
19
19
|
info_hash = {"language" => "ruby", "name" => flow._name}
|
20
20
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, _info_file)
|
21
21
|
end
|
22
22
|
|
23
|
-
# Get the
|
24
|
-
|
25
|
-
yield(
|
26
|
-
|
27
|
-
|
23
|
+
# Get the source-specific logic...
|
24
|
+
source = Zillabyte::Harness::Source.new(false)
|
25
|
+
#yield(source)
|
26
|
+
source.instance_eval(&block)
|
27
|
+
source._name = "source"
|
28
|
+
Zillabyte::Harness::Helper.check_source("simple_source", source)
|
29
|
+
flow._nodes << source
|
28
30
|
if(options[:command] == :info)
|
29
|
-
info_hash = {"name" =>
|
30
|
-
if(
|
31
|
-
info_hash["emits"] =
|
32
|
-
elsif(
|
33
|
-
info_hash["relation"] =
|
34
|
-
elsif(
|
35
|
-
info_hash["matches"] =
|
31
|
+
info_hash = {"name" => source._name, "type" => source._type}
|
32
|
+
if(source._emits)
|
33
|
+
info_hash["emits"] = source._emits
|
34
|
+
elsif(source._relation)
|
35
|
+
info_hash["relation"] = source._relation
|
36
|
+
elsif(source._matches)
|
37
|
+
info_hash["matches"] = source._matches
|
36
38
|
end
|
37
39
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, _info_file)
|
38
40
|
end
|
@@ -40,14 +42,14 @@ class Zillabyte::Harness::SimpleSpout
|
|
40
42
|
# Add the sink...
|
41
43
|
flow._nodes << flow._sink
|
42
44
|
if(options[:command] == :info)
|
43
|
-
info_hash = {"name" => flow._sink._name, "type" => flow._sink._type, "columns" => flow._sink._columns}
|
45
|
+
info_hash = {"name" => flow._sink._name, "type" => flow._sink._type, "columns" => flow._sink._columns, "relation" => flow._sink._relation || flow._sink._name}
|
44
46
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, _info_file)
|
45
47
|
end
|
46
48
|
|
47
49
|
# Execute...
|
48
|
-
if(options[:command] == :execute and options[:name] == "
|
50
|
+
if(options[:command] == :execute and options[:name] == "source")
|
49
51
|
pipe_name = options[:pipe]
|
50
|
-
c = Zillabyte::Harness::
|
52
|
+
c = Zillabyte::Harness::SourceController.new(source, progress = Zillabyte::Common::Progress.new)
|
51
53
|
c.run(pipe_name)
|
52
54
|
end
|
53
55
|
|
@@ -58,16 +60,17 @@ class Zillabyte::Harness::SimpleSpout
|
|
58
60
|
|
59
61
|
|
60
62
|
def name(v)
|
61
|
-
Zillabyte::Harness::Helper.check_name("
|
63
|
+
Zillabyte::Harness::Helper.check_name("simple_source", v, {})
|
62
64
|
@_name = v
|
63
65
|
end
|
64
66
|
|
65
67
|
|
66
68
|
def emits(v)
|
67
69
|
# Construct the sink...
|
68
|
-
Zillabyte::Harness::Helper.check_emits("
|
70
|
+
Zillabyte::Harness::Helper.check_emits("simple_source", v, {})
|
69
71
|
@_sink = Zillabyte::Harness::Sink.new()
|
70
|
-
@_sink.
|
72
|
+
@_sink._name = "sink"
|
73
|
+
@_sink._relation = v[0][0]
|
71
74
|
columns = v[0][1]
|
72
75
|
columns.each do |col|
|
73
76
|
col.each do |cname, ctype|
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::Sink
|
2
|
-
attr_accessor :_name, :_type, :_columns, :_consumes
|
2
|
+
attr_accessor :_name, :_relation, :_type, :_columns, :_consumes
|
3
3
|
|
4
4
|
def initialize()
|
5
5
|
@_type = 'sink'
|
@@ -15,6 +15,10 @@ class Zillabyte::Harness::Sink
|
|
15
15
|
@_name = v
|
16
16
|
end
|
17
17
|
|
18
|
+
def relation(v)
|
19
|
+
@_relation = v
|
20
|
+
end
|
21
|
+
|
18
22
|
def column(cname, ctype)
|
19
23
|
col = {cname => ctype}
|
20
24
|
@_columns << col
|
@@ -1,9 +1,9 @@
|
|
1
|
-
class Zillabyte::Harness::
|
1
|
+
class Zillabyte::Harness::Source
|
2
2
|
attr_accessor :_name, :_type, :_matches, :_relation, :_emits, :_prepare, :_next_batch, :_flow_flag
|
3
3
|
|
4
4
|
def initialize(flow_flag)
|
5
|
-
@_name = "
|
6
|
-
@_type = '
|
5
|
+
@_name = "source_"+Zillabyte::Harness::Counter.get()
|
6
|
+
@_type = 'source'
|
7
7
|
@_flow_flag = flow_flag
|
8
8
|
end
|
9
9
|
|
@@ -20,10 +20,12 @@ class Zillabyte::Harness::Topology
|
|
20
20
|
h
|
21
21
|
end
|
22
22
|
|
23
|
-
def
|
24
|
-
h = Zillabyte::Harness::
|
25
|
-
yield(h)
|
26
|
-
|
23
|
+
def source(&block)
|
24
|
+
h = Zillabyte::Harness::Source.new(true)
|
25
|
+
#yield(h)
|
26
|
+
h.instance_eval(&block)
|
27
|
+
Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
|
28
|
+
Zillabyte::Harness::Helper.check_source("source", h)
|
27
29
|
@_nodes << h
|
28
30
|
if(@_options[:command] == :info)
|
29
31
|
info_hash = {"name" => h._name, "type" => h._type}
|
@@ -31,30 +33,22 @@ class Zillabyte::Harness::Topology
|
|
31
33
|
info_hash["relation"] = h._relation
|
32
34
|
elsif(h._matches)
|
33
35
|
info_hash["matches"] = h._matches
|
36
|
+
elsif(h._emits)
|
37
|
+
@_branched = Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams) || @_branched
|
38
|
+
info_hash["emits"] = h._emits
|
34
39
|
end
|
35
40
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
|
39
|
-
def spout
|
40
|
-
h = Zillabyte::Harness::Spout.new(true)
|
41
|
-
yield(h)
|
42
|
-
Zillabyte::Harness::Helper.check_name("spout", h._name, @_names)
|
43
|
-
@_branched = Zillabyte::Harness::Helper.check_emits("spout", h._emits, @_streams) || @_branched
|
44
|
-
@_nodes << h
|
45
|
-
if(@_options[:command] == :info)
|
46
|
-
info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
|
47
|
-
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
|
48
41
|
elsif(@_options[:command] == :execute and @_options[:name] == h._name)
|
49
42
|
pipe_name = @_options[:pipe]
|
50
|
-
c = Zillabyte::Harness::
|
43
|
+
c = Zillabyte::Harness::SourceController.new(h, progress = Zillabyte::Common::Progress.new)
|
51
44
|
c.run(pipe_name)
|
52
45
|
end
|
53
46
|
end
|
54
47
|
|
55
|
-
def each
|
48
|
+
def each(&block)
|
56
49
|
h = Zillabyte::Harness::Each.new()
|
57
|
-
yield(h)
|
50
|
+
#yield(h)
|
51
|
+
h.instance_eval(&block)
|
58
52
|
Zillabyte::Harness::Helper.check_name("each", h._name, @_names)
|
59
53
|
if(@_branched)
|
60
54
|
Zillabyte::Harness::Helper.check_consumes(h, @_streams)
|
@@ -78,9 +72,10 @@ class Zillabyte::Harness::Topology
|
|
78
72
|
end
|
79
73
|
end
|
80
74
|
|
81
|
-
def aggregate
|
75
|
+
def aggregate(&block)
|
82
76
|
h = Zillabyte::Harness::Aggregate.new()
|
83
|
-
yield(h)
|
77
|
+
#yield(h)
|
78
|
+
h.instance_eval(&block)
|
84
79
|
Zillabyte::Harness::Helper.check_name("aggregate", h._name, @_names)
|
85
80
|
if(@_branched)
|
86
81
|
Zillabyte::Harness::Helper.check_consumes(h, @_streams)
|
@@ -107,9 +102,10 @@ class Zillabyte::Harness::Topology
|
|
107
102
|
end
|
108
103
|
end
|
109
104
|
|
110
|
-
def sink
|
105
|
+
def sink(&block)
|
111
106
|
h = Zillabyte::Harness::Sink.new()
|
112
|
-
yield(h)
|
107
|
+
#yield(h)
|
108
|
+
h.instance_eval(&block)
|
113
109
|
Zillabyte::Harness::Helper.check_sink(h, @_nodes)
|
114
110
|
if(@_branched)
|
115
111
|
Zillabyte::Harness::Helper.check_consumes(h, @_streams)
|
@@ -120,7 +116,7 @@ class Zillabyte::Harness::Topology
|
|
120
116
|
end
|
121
117
|
@_nodes << h
|
122
118
|
if(@_options[:command] == :info)
|
123
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns}
|
119
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name}
|
124
120
|
if(h._consumes)
|
125
121
|
info_hash["consumes"] = h._consumes
|
126
122
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-04-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -68,9 +68,9 @@ files:
|
|
68
68
|
- ruby/lib/zillabyte/harness/live_delegator.rb
|
69
69
|
- ruby/lib/zillabyte/harness/simple_aggregate.rb
|
70
70
|
- ruby/lib/zillabyte/harness/simple_function.rb
|
71
|
-
- ruby/lib/zillabyte/harness/
|
71
|
+
- ruby/lib/zillabyte/harness/simple_source.rb
|
72
72
|
- ruby/lib/zillabyte/harness/sink.rb
|
73
|
-
- ruby/lib/zillabyte/harness/
|
73
|
+
- ruby/lib/zillabyte/harness/source.rb
|
74
74
|
- ruby/lib/zillabyte/harness/topology.rb
|
75
75
|
- ruby/lib/zillabyte/harness/tuple.rb
|
76
76
|
- ruby/lib/zillabyte/harness.rb
|
@@ -97,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
97
97
|
version: '0'
|
98
98
|
requirements: []
|
99
99
|
rubyforge_project:
|
100
|
-
rubygems_version: 2.
|
100
|
+
rubygems_version: 2.0.7
|
101
101
|
signing_key:
|
102
102
|
specification_version: 4
|
103
103
|
summary: The Official Zillabyte Gem
|