zillabyte 0.1.44 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ruby/lib/zillabyte/harness/app.rb +3 -2
- data/ruby/lib/zillabyte/harness/base.rb +1 -1
- data/ruby/lib/zillabyte/harness/common_node.rb +5 -1
- data/ruby/lib/zillabyte/harness/component.rb +4 -3
- data/ruby/lib/zillabyte/harness/{component_source.rb → component_input.rb} +2 -2
- data/ruby/lib/zillabyte/harness/{component_sink.rb → component_output.rb} +2 -2
- data/ruby/lib/zillabyte/harness/component_stream.rb +1 -1
- data/ruby/lib/zillabyte/harness/each.rb +5 -8
- data/ruby/lib/zillabyte/harness/filter.rb +5 -8
- data/ruby/lib/zillabyte/harness/group_by.rb +3 -2
- data/ruby/lib/zillabyte/harness/helper.rb +28 -6
- data/ruby/lib/zillabyte/harness/join.rb +2 -2
- data/ruby/lib/zillabyte/harness/live_delegator.rb +39 -33
- data/ruby/lib/zillabyte/harness/operation_handler.rb +6 -6
- data/ruby/lib/zillabyte/harness/source.rb +5 -7
- data/ruby/lib/zillabyte/harness/stream.rb +14 -5
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +6 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1af0e7dd6900cca31d707e201cfbdfcc8e4fe3f3
|
4
|
+
data.tar.gz: 37532e5dcb0754700d339bf0c4863c7161098983
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42a4e7ecf444f7cc461d2175d2a6a7ecb8658de9452a1c1b91a202c6060fba9e561cc766fbe0edfa941d8a14dfaabf99e333449a38ebd9dbbf938b9a5700654f
|
7
|
+
data.tar.gz: 137d82edc66b7950b671989b3a4631d1e39e88606442130c8ad8a5f389f0da717743725cb6e0f01160732497e04cb8eb5b39afd4b8e8928697a0709ff08a266b
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "json"
|
2
2
|
require "zillabyte/harness/base"
|
3
|
+
require "socket"
|
3
4
|
|
4
5
|
class Zillabyte::Harness::App < Zillabyte::Harness::Base
|
5
6
|
|
@@ -8,9 +9,9 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
|
|
8
9
|
Zillabyte::Harness::Helper.check_name("app", h._name, {})
|
9
10
|
h._options = Zillabyte::Harness::Helper.opt_parser()
|
10
11
|
if(h._options[:command] == :info)
|
11
|
-
h.
|
12
|
+
h._socket = TCPSocket.new(h._options[:host], h._options[:port])
|
12
13
|
info_hash = {"language" => "ruby", "name" => h._name, "flow_type" => "app", "multilang_version" => Zillabyte::VERSION}
|
13
|
-
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h.
|
14
|
+
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._socket)
|
14
15
|
end
|
15
16
|
h
|
16
17
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::CommonNode
|
2
|
-
attr_accessor :_name, :_type, :_emits, :_parallelism
|
2
|
+
attr_accessor :_name, :_type, :_emits, :_parallelism, :_prepare
|
3
3
|
|
4
4
|
def name(v)
|
5
5
|
@_name = v
|
@@ -13,4 +13,8 @@ class Zillabyte::Harness::CommonNode
|
|
13
13
|
@_parallelism = v
|
14
14
|
end
|
15
15
|
|
16
|
+
def prepare(&block)
|
17
|
+
@_prepare = block
|
18
|
+
end
|
19
|
+
|
16
20
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require "zillabyte/harness/base"
|
2
|
+
require "socket"
|
2
3
|
|
3
4
|
class Zillabyte::Harness::Component < Zillabyte::Harness::Base
|
4
5
|
|
@@ -7,16 +8,16 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
|
|
7
8
|
Zillabyte::Harness::Helper.check_name("component", h._name, {})
|
8
9
|
h._options = Zillabyte::Harness::Helper.opt_parser()
|
9
10
|
if(h._options[:command] == :info)
|
10
|
-
h.
|
11
|
+
h._socket = TCPSocket.new(h._options[:host], h._options[:port])
|
11
12
|
info_hash = {"language" => "ruby", "name" => h._name, "flow_type" => "component", "multilang_version" => Zillabyte::VERSION}
|
12
|
-
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h.
|
13
|
+
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._socket)
|
13
14
|
end
|
14
15
|
h
|
15
16
|
end
|
16
17
|
|
17
18
|
def inputs(*args, &block)
|
18
19
|
op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::ComponentStream)
|
19
|
-
op.build_multilang_operation("
|
20
|
+
op.build_multilang_operation("component_input", *args, &block)
|
20
21
|
.add_operation_properties_to_info(:name, :type, :fields)
|
21
22
|
.handle_operation
|
22
23
|
.get_output_streams
|
@@ -1,11 +1,11 @@
|
|
1
|
-
class Zillabyte::Harness::
|
1
|
+
class Zillabyte::Harness::ComponentInput
|
2
2
|
attr_accessor :_app, :_node
|
3
3
|
|
4
4
|
class Node
|
5
5
|
attr_accessor :_name, :_type, :_emits, :_fields
|
6
6
|
|
7
7
|
def initialize()
|
8
|
-
@_type = "
|
8
|
+
@_type = "input"
|
9
9
|
@_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
10
10
|
@_fields = []
|
11
11
|
end
|
@@ -1,11 +1,11 @@
|
|
1
|
-
class Zillabyte::Harness::
|
1
|
+
class Zillabyte::Harness::ComponentOutput
|
2
2
|
attr_accessor :_app, :_node
|
3
3
|
|
4
4
|
class Node
|
5
5
|
attr_accessor :_name, :_relation, :_type, :_columns, :_scope
|
6
6
|
|
7
7
|
def initialize()
|
8
|
-
@_type = '
|
8
|
+
@_type = 'output'
|
9
9
|
@_columns = []
|
10
10
|
end
|
11
11
|
|
@@ -4,7 +4,7 @@ class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
|
|
4
4
|
|
5
5
|
def outputs(*args, &block)
|
6
6
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
7
|
-
op.build_multilang_operation("
|
7
|
+
op.build_multilang_operation("component_output", *args, &block)
|
8
8
|
.add_operation_properties_to_info(:name, :type, :columns, :scope)
|
9
9
|
.add_optional_operation_properties_to_info(:relation)
|
10
10
|
.create_arc_info_from_stream(self)
|
@@ -2,7 +2,7 @@ class Zillabyte::Harness::Each
|
|
2
2
|
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
4
|
class Node < Zillabyte::Harness::CommonNode
|
5
|
-
attr_accessor :_output_format, :
|
5
|
+
attr_accessor :_output_format, :_execute
|
6
6
|
|
7
7
|
def initialize()
|
8
8
|
@_name = "each_"+Zillabyte::Harness::Counter.get()
|
@@ -14,10 +14,6 @@ class Zillabyte::Harness::Each
|
|
14
14
|
@_output_format = v
|
15
15
|
end
|
16
16
|
|
17
|
-
def prepare(&block)
|
18
|
-
@_prepare = block
|
19
|
-
end
|
20
|
-
|
21
17
|
def execute(&block)
|
22
18
|
@_execute = block
|
23
19
|
end
|
@@ -49,8 +45,9 @@ class Zillabyte::Harness::Each
|
|
49
45
|
end
|
50
46
|
|
51
47
|
def run_operation
|
52
|
-
|
53
|
-
|
54
|
-
c.
|
48
|
+
host = @_app._options[:host]
|
49
|
+
port = @_app._options[:port]
|
50
|
+
c = Zillabyte::Harness::EachController.new(@_node, Zillabyte::Common::Progress.new)
|
51
|
+
c.run(host, port)
|
55
52
|
end
|
56
53
|
end
|
@@ -2,17 +2,13 @@ class Zillabyte::Harness::Filter
|
|
2
2
|
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
4
|
class Node < Zillabyte::Harness::CommonNode
|
5
|
-
attr_accessor :
|
5
|
+
attr_accessor :_keep
|
6
6
|
|
7
7
|
def initialize()
|
8
8
|
@_name = "filter_"+Zillabyte::Harness::Counter.get()
|
9
9
|
@_type = 'filter'
|
10
10
|
end
|
11
11
|
|
12
|
-
def prepare(&block)
|
13
|
-
@_prepare = block
|
14
|
-
end
|
15
|
-
|
16
12
|
def keep(&block)
|
17
13
|
@_keep = block
|
18
14
|
end
|
@@ -44,9 +40,10 @@ class Zillabyte::Harness::Filter
|
|
44
40
|
end
|
45
41
|
|
46
42
|
def run_operation
|
47
|
-
|
48
|
-
|
49
|
-
c.
|
43
|
+
host = @_app._options[:host]
|
44
|
+
port = @_app._options[:port]
|
45
|
+
c = Zillabyte::Harness::FilterController.new(@_node, Zillabyte::Common::Progress.new)
|
46
|
+
c.run(host, port)
|
50
47
|
end
|
51
48
|
|
52
49
|
end
|
@@ -47,8 +47,9 @@ class Zillabyte::Harness::GroupBy
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def run_operation
|
50
|
-
|
50
|
+
host = @_app._options[:host]
|
51
|
+
port = @_app._options[:port]
|
51
52
|
c = Zillabyte::Harness::GroupByController.new(@_node, Zillabyte::Common::Progress.new)
|
52
|
-
c.run(
|
53
|
+
c.run(host, port)
|
53
54
|
end
|
54
55
|
end
|
@@ -5,6 +5,7 @@ class Zillabyte::Harness::Helper
|
|
5
5
|
META_NAMES = ["id", "confidence", "since", "source"]
|
6
6
|
ALLOWED_OUTPUT_FORMATS = [:replace, :merge]
|
7
7
|
ALLOWED_END_CYCLE_POLICIES = [:null_emit, :explicit, :infinite]
|
8
|
+
ALLOWED_TYPES = [:string, :integer, :float, :double, :boolean, :array, :map]
|
8
9
|
|
9
10
|
def self.opt_parser()
|
10
11
|
options = {
|
@@ -20,8 +21,11 @@ class Zillabyte::Harness::Helper
|
|
20
21
|
opts.on("--name NAME") do |v|
|
21
22
|
options[:name] = v
|
22
23
|
end
|
23
|
-
opts.on("--
|
24
|
-
options[:
|
24
|
+
opts.on("--host HOST") do |v|
|
25
|
+
options[:host] = v
|
26
|
+
end
|
27
|
+
opts.on("--port PORT") do |v|
|
28
|
+
options[:port] = v
|
25
29
|
end
|
26
30
|
opts.on("--file FNAME") do |v|
|
27
31
|
options[:file] = v
|
@@ -343,7 +347,7 @@ class Zillabyte::Harness::Helper
|
|
343
347
|
msg = "#{ee}Field data types must be SYMBOLS in \"#{relation_name}\". #{pp}"
|
344
348
|
Zillabyte::Harness::Helper.print_error(msg)
|
345
349
|
end
|
346
|
-
if(ctype
|
350
|
+
if(!ALLOWED_TYPES.member?(ctype))
|
347
351
|
msg = "#{ee}Invalid field data type at \"#{ctype}\" in \"#{relation_name}\". #{pp}"
|
348
352
|
Zillabyte::Harness::Helper.print_error(msg)
|
349
353
|
end
|
@@ -382,6 +386,24 @@ class Zillabyte::Harness::Helper
|
|
382
386
|
Zillabyte::Harness::Helper.check_field_format("outputs", pp, cname, ctype, node._name)
|
383
387
|
end
|
384
388
|
end
|
389
|
+
|
390
|
+
def self.check_loop_back(node_name, nodes)
|
391
|
+
ee = "Error in \"loop_back\": \n\t "
|
392
|
+
found = false
|
393
|
+
nodes.each do |node|
|
394
|
+
if node._name == node_name
|
395
|
+
found = true
|
396
|
+
if node._type == "source"
|
397
|
+
msg = "#{ee}Cannot loop back to source node \"#{node_name}\"!"
|
398
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
399
|
+
end
|
400
|
+
end
|
401
|
+
end
|
402
|
+
if !found
|
403
|
+
msg = "#{ee}The specified loop-back node \"#{node_name}\" was not found in the operations preceding it."
|
404
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
405
|
+
end
|
406
|
+
end
|
385
407
|
|
386
408
|
def self.get_non_option_args(args)
|
387
409
|
if args.nil?
|
@@ -422,7 +444,7 @@ class Zillabyte::Harness::Helper
|
|
422
444
|
- "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
423
445
|
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
424
446
|
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
425
|
-
- Field types must be SYMBOLS. The following types are allowed
|
447
|
+
- Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
|
426
448
|
OUTPUT
|
427
449
|
|
428
450
|
@@_print_check_source = <<-OUTPUT
|
@@ -530,7 +552,7 @@ OUTPUT
|
|
530
552
|
- "Inputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
531
553
|
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
532
554
|
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
533
|
-
- Field types must be SYMBOLS. The following types are allowed
|
555
|
+
- Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
|
534
556
|
OUTPUT
|
535
557
|
|
536
558
|
@@_print_check_component_sink = <<-OUTPUT
|
@@ -543,7 +565,7 @@ OUTPUT
|
|
543
565
|
- "Outputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
544
566
|
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
545
567
|
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
546
|
-
- Field types must be SYMBOLS. The following types are allowed
|
568
|
+
- Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
|
547
569
|
OUTPUT
|
548
570
|
|
549
571
|
@@_print_check_call_component = <<-OUTPUT
|
@@ -8,8 +8,8 @@ class Zillabyte::Harness::Join
|
|
8
8
|
@_name = "join_"+Zillabyte::Harness::Counter.get()
|
9
9
|
@_type = 'join'
|
10
10
|
@_join_type = :inner
|
11
|
-
@_lhs_stream = args[0]
|
12
|
-
@_rhs_stream = args[1]
|
11
|
+
@_lhs_stream = args[0]._name
|
12
|
+
@_rhs_stream = args[1]._name
|
13
13
|
options = args[2]
|
14
14
|
|
15
15
|
options.symbolize_keys!
|
@@ -18,24 +18,13 @@ module Zillabyte
|
|
18
18
|
DEATH_TIMEOUT = READ_SOFT_TIMEOUT * 6 # If no activity from the parent (even response to a ping) time before we kill ourself.
|
19
19
|
|
20
20
|
class << self
|
21
|
-
attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :
|
21
|
+
attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :host, :port, :socket
|
22
22
|
end
|
23
23
|
|
24
24
|
self.pending_commands = []
|
25
25
|
|
26
26
|
def setup_pipes
|
27
|
-
|
28
|
-
if(pipe_name and File.exist?(pipe_name+".in"))
|
29
|
-
Storm::Protocol.pipe_to_java = File.open("#{pipe_name}.in","w+")
|
30
|
-
else
|
31
|
-
Storm::Protocol.pipe_to_java = $stdout
|
32
|
-
end
|
33
|
-
|
34
|
-
if(pipe_name and File.exist?(pipe_name+".out"))
|
35
|
-
Storm::Protocol.pipe_from_java = File.open("#{pipe_name}.out","r+")
|
36
|
-
else
|
37
|
-
Storm::Protocol.pipe_from_java = $stdin
|
38
|
-
end
|
27
|
+
Storm::Protocol.socket = TCPSocket.open(Storm::Protocol.host || "0.0.0.0", Storm::Protocol.port)
|
39
28
|
end
|
40
29
|
|
41
30
|
def read_message
|
@@ -46,7 +35,7 @@ module Zillabyte
|
|
46
35
|
loop do
|
47
36
|
|
48
37
|
# Read the next line.. The IO.select magic here is a way to minimize concurrency overhead
|
49
|
-
results = IO.select([Storm::Protocol.
|
38
|
+
results = IO.select([Storm::Protocol.socket], nil, nil, READ_SOFT_TIMEOUT)
|
50
39
|
|
51
40
|
# Process timeout...
|
52
41
|
if results.nil?
|
@@ -102,9 +91,8 @@ module Zillabyte
|
|
102
91
|
def send_msg_to_parent(msg)
|
103
92
|
# log msg
|
104
93
|
# log msg.to_json
|
105
|
-
Storm::Protocol.
|
106
|
-
Storm::Protocol.
|
107
|
-
Storm::Protocol.pipe_to_java.flush
|
94
|
+
Storm::Protocol.socket.write(msg.to_json+"\n")
|
95
|
+
Storm::Protocol.socket.write("end\n")
|
108
96
|
end
|
109
97
|
|
110
98
|
def send_pid(heartbeat_dir)
|
@@ -219,9 +207,10 @@ module Zillabyte
|
|
219
207
|
|
220
208
|
def ack(id); end
|
221
209
|
|
222
|
-
def run(
|
210
|
+
def run(host, port)
|
223
211
|
Storm::Protocol.mode = 'source'
|
224
|
-
Storm::Protocol.
|
212
|
+
Storm::Protocol.host = host
|
213
|
+
Storm::Protocol.port = port
|
225
214
|
setup_pipes
|
226
215
|
handshake
|
227
216
|
|
@@ -262,19 +251,24 @@ module Zillabyte
|
|
262
251
|
|
263
252
|
def execute(tuple); end
|
264
253
|
|
265
|
-
def run(
|
254
|
+
def run(host, port)
|
266
255
|
Storm::Protocol.mode = 'each'
|
267
|
-
Storm::Protocol.
|
256
|
+
Storm::Protocol.host = host
|
257
|
+
Storm::Protocol.port = port
|
268
258
|
setup_pipes
|
269
|
-
|
259
|
+
handshake
|
270
260
|
|
271
261
|
while true
|
272
262
|
begin
|
273
263
|
m = read_command()
|
274
264
|
if(m)
|
275
|
-
|
276
|
-
|
277
|
-
|
265
|
+
if m['command'] && m['command'] == 'prepare'
|
266
|
+
prepare()
|
267
|
+
else
|
268
|
+
t = Tuple.from_hash(m)
|
269
|
+
if(t)
|
270
|
+
execute t
|
271
|
+
end
|
278
272
|
end
|
279
273
|
end
|
280
274
|
rescue SignalException => e
|
@@ -297,19 +291,24 @@ module Zillabyte
|
|
297
291
|
|
298
292
|
def keep(tuple); end
|
299
293
|
|
300
|
-
def run(
|
294
|
+
def run(host, port)
|
301
295
|
Storm::Protocol.mode = 'filter'
|
302
|
-
Storm::Protocol.
|
296
|
+
Storm::Protocol.host = host
|
297
|
+
Storm::Protocol.port = port
|
303
298
|
setup_pipes
|
304
|
-
|
299
|
+
handshake
|
305
300
|
|
306
301
|
while true
|
307
302
|
begin
|
308
303
|
m = read_command()
|
309
304
|
if m
|
310
|
-
|
311
|
-
|
312
|
-
|
305
|
+
if m['command'] && m['command'] == 'prepare'
|
306
|
+
prepare()
|
307
|
+
else
|
308
|
+
t = Tuple.from_hash(m)
|
309
|
+
if t
|
310
|
+
emit(t) if keep t
|
311
|
+
end
|
313
312
|
end
|
314
313
|
end
|
315
314
|
rescue SignalException => e
|
@@ -326,9 +325,10 @@ module Zillabyte
|
|
326
325
|
class GroupBy
|
327
326
|
include Storm::Protocol
|
328
327
|
|
329
|
-
def run(
|
328
|
+
def run(host, port )
|
330
329
|
Storm::Protocol.mode = 'group_by'
|
331
|
-
Storm::Protocol.
|
330
|
+
Storm::Protocol.host = host
|
331
|
+
Storm::Protocol.port = port
|
332
332
|
setup_pipes
|
333
333
|
handshake
|
334
334
|
|
@@ -342,6 +342,8 @@ module Zillabyte
|
|
342
342
|
when 'aggregate'
|
343
343
|
t = Tuple.from_hash(msg)
|
344
344
|
aggregate t
|
345
|
+
when 'prepare'
|
346
|
+
prepare
|
345
347
|
when 'end_group'
|
346
348
|
end_group
|
347
349
|
end
|
@@ -432,6 +434,10 @@ module Zillabyte
|
|
432
434
|
def begin_group(*args)
|
433
435
|
self.instance_exec *args, &@harness._begin_group
|
434
436
|
end
|
437
|
+
|
438
|
+
def prepare(*args)
|
439
|
+
self.instance_exec *args, &@harness._prepare if @harness._prepare
|
440
|
+
end
|
435
441
|
|
436
442
|
def aggregate(*args)
|
437
443
|
self.instance_exec *args, &@harness._aggregate
|
@@ -30,10 +30,10 @@ class Zillabyte::Harness::OperationHandler
|
|
30
30
|
@_operation = Zillabyte::Harness::InjectedComponent.new(@_app, *args)
|
31
31
|
when "sink"
|
32
32
|
@_operation = Zillabyte::Harness::Sink.new(@_app, *args)
|
33
|
-
when "
|
34
|
-
@_operation = Zillabyte::Harness::
|
35
|
-
when "
|
36
|
-
@_operation = Zillabyte::Harness::
|
33
|
+
when "component_input"
|
34
|
+
@_operation = Zillabyte::Harness::ComponentInput.new(@_app, *args)
|
35
|
+
when "component_output"
|
36
|
+
@_operation = Zillabyte::Harness::ComponentOutput.new(@_app, *args)
|
37
37
|
else
|
38
38
|
throw "Unknown type"
|
39
39
|
end
|
@@ -93,13 +93,13 @@ class Zillabyte::Harness::OperationHandler
|
|
93
93
|
@_node_hash_optpars.each do |par|
|
94
94
|
node_hash[par] = node.instance_variable_get("@_#{par}") if node.instance_variable_defined?("@_#{par}")
|
95
95
|
end
|
96
|
-
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app.
|
96
|
+
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._socket)
|
97
97
|
|
98
98
|
# Create arc hashes from streams that feed into this node
|
99
99
|
@_arc_hash_streams.each do |stream, direction|
|
100
100
|
arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => node._name}
|
101
101
|
arc_hash[direction] = 1 if !direction.nil?
|
102
|
-
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app.
|
102
|
+
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
|
103
103
|
end
|
104
104
|
elsif(@_app._options[:command] == :execute and @_app._options[:name] == node._name)
|
105
105
|
@_operation.run_operation
|
@@ -2,7 +2,7 @@ class Zillabyte::Harness::Source
|
|
2
2
|
attr_accessor :_app, :_node, :_relation
|
3
3
|
|
4
4
|
class Node < Zillabyte::Harness::CommonNode
|
5
|
-
attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
|
5
|
+
attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
|
6
6
|
|
7
7
|
def initialize()
|
8
8
|
@_name = "source_"+Zillabyte::Harness::Counter.get()
|
@@ -35,9 +35,6 @@ class Zillabyte::Harness::Source
|
|
35
35
|
@_next_tuple = block
|
36
36
|
end
|
37
37
|
|
38
|
-
def prepare(&block)
|
39
|
-
@_prepare = block
|
40
|
-
end
|
41
38
|
end
|
42
39
|
|
43
40
|
def initialize(app, *args)
|
@@ -66,8 +63,9 @@ class Zillabyte::Harness::Source
|
|
66
63
|
end
|
67
64
|
|
68
65
|
def run_operation
|
69
|
-
|
70
|
-
|
71
|
-
c.
|
66
|
+
host = @_app._options[:host]
|
67
|
+
port = @_app._options[:port]
|
68
|
+
c = Zillabyte::Harness::SourceController.new(@_node, Zillabyte::Common::Progress.new)
|
69
|
+
c.run(host, port)
|
72
70
|
end
|
73
71
|
end
|
@@ -45,12 +45,13 @@ class Zillabyte::Harness::Stream
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def join_with(*args)
|
48
|
+
lhs_stream = self
|
49
|
+
rhs_stream = args[0]
|
48
50
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
49
|
-
op
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
.create_arc_info_from_stream(node._rhs_stream, :right)
|
51
|
+
op.build_multilang_operation("join", self, *args)
|
52
|
+
.add_operation_properties_to_info(:name, :type, :lhs_fields, :rhs_fields, :join_type, :lhs_stream, :rhs_stream)
|
53
|
+
.create_arc_info_from_stream(lhs_stream, :left)
|
54
|
+
.create_arc_info_from_stream(rhs_stream, :right)
|
54
55
|
.handle_operation
|
55
56
|
.get_output_streams
|
56
57
|
end
|
@@ -69,6 +70,14 @@ class Zillabyte::Harness::Stream
|
|
69
70
|
alias_method :executes, :call_component
|
70
71
|
alias_method :execute, :call_component
|
71
72
|
|
73
|
+
def loop_back(*args, &block)
|
74
|
+
# This is not a real operation, just telling the stream to loop back to the previous operation
|
75
|
+
loop_back_node = args[0]
|
76
|
+
Zillabyte::Harness::Helper.check_loop_back(loop_back_node, @_app._nodes)
|
77
|
+
Zillabyte::Harness::Helper.write_arc_to_file({"name" => self._name, "origin" => self._previous_node_name, "dest" => loop_back_node, "loop_back" => 1}, @_app._socket)
|
78
|
+
self
|
79
|
+
end
|
80
|
+
|
72
81
|
def sink(*args, &block)
|
73
82
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
74
83
|
op.build_multilang_operation("sink", *args, &block)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-09-
|
11
|
+
date: 2014-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -24,34 +24,20 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: mkfifo
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: zillabyte-cli
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
44
30
|
requirements:
|
45
31
|
- - "~>"
|
46
32
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
33
|
+
version: 0.9.0
|
48
34
|
type: :runtime
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|
52
38
|
- - "~>"
|
53
39
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
40
|
+
version: 0.9.0
|
55
41
|
description: The Official Zillabyte Gem
|
56
42
|
email:
|
57
43
|
- gem@zillabyte.com
|
@@ -67,8 +53,8 @@ files:
|
|
67
53
|
- ruby/lib/zillabyte/harness/clump.rb
|
68
54
|
- ruby/lib/zillabyte/harness/common_node.rb
|
69
55
|
- ruby/lib/zillabyte/harness/component.rb
|
70
|
-
- ruby/lib/zillabyte/harness/
|
71
|
-
- ruby/lib/zillabyte/harness/
|
56
|
+
- ruby/lib/zillabyte/harness/component_input.rb
|
57
|
+
- ruby/lib/zillabyte/harness/component_output.rb
|
72
58
|
- ruby/lib/zillabyte/harness/component_stream.rb
|
73
59
|
- ruby/lib/zillabyte/harness/counter.rb
|
74
60
|
- ruby/lib/zillabyte/harness/each.rb
|