zillabyte 0.1.44 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ruby/lib/zillabyte/harness/app.rb +3 -2
- data/ruby/lib/zillabyte/harness/base.rb +1 -1
- data/ruby/lib/zillabyte/harness/common_node.rb +5 -1
- data/ruby/lib/zillabyte/harness/component.rb +4 -3
- data/ruby/lib/zillabyte/harness/{component_source.rb → component_input.rb} +2 -2
- data/ruby/lib/zillabyte/harness/{component_sink.rb → component_output.rb} +2 -2
- data/ruby/lib/zillabyte/harness/component_stream.rb +1 -1
- data/ruby/lib/zillabyte/harness/each.rb +5 -8
- data/ruby/lib/zillabyte/harness/filter.rb +5 -8
- data/ruby/lib/zillabyte/harness/group_by.rb +3 -2
- data/ruby/lib/zillabyte/harness/helper.rb +28 -6
- data/ruby/lib/zillabyte/harness/join.rb +2 -2
- data/ruby/lib/zillabyte/harness/live_delegator.rb +39 -33
- data/ruby/lib/zillabyte/harness/operation_handler.rb +6 -6
- data/ruby/lib/zillabyte/harness/source.rb +5 -7
- data/ruby/lib/zillabyte/harness/stream.rb +14 -5
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +6 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1af0e7dd6900cca31d707e201cfbdfcc8e4fe3f3
|
4
|
+
data.tar.gz: 37532e5dcb0754700d339bf0c4863c7161098983
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 42a4e7ecf444f7cc461d2175d2a6a7ecb8658de9452a1c1b91a202c6060fba9e561cc766fbe0edfa941d8a14dfaabf99e333449a38ebd9dbbf938b9a5700654f
|
7
|
+
data.tar.gz: 137d82edc66b7950b671989b3a4631d1e39e88606442130c8ad8a5f389f0da717743725cb6e0f01160732497e04cb8eb5b39afd4b8e8928697a0709ff08a266b
|
@@ -1,5 +1,6 @@
|
|
1
1
|
require "json"
|
2
2
|
require "zillabyte/harness/base"
|
3
|
+
require "socket"
|
3
4
|
|
4
5
|
class Zillabyte::Harness::App < Zillabyte::Harness::Base
|
5
6
|
|
@@ -8,9 +9,9 @@ class Zillabyte::Harness::App < Zillabyte::Harness::Base
|
|
8
9
|
Zillabyte::Harness::Helper.check_name("app", h._name, {})
|
9
10
|
h._options = Zillabyte::Harness::Helper.opt_parser()
|
10
11
|
if(h._options[:command] == :info)
|
11
|
-
h.
|
12
|
+
h._socket = TCPSocket.new(h._options[:host], h._options[:port])
|
12
13
|
info_hash = {"language" => "ruby", "name" => h._name, "flow_type" => "app", "multilang_version" => Zillabyte::VERSION}
|
13
|
-
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h.
|
14
|
+
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._socket)
|
14
15
|
end
|
15
16
|
h
|
16
17
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::CommonNode
|
2
|
-
attr_accessor :_name, :_type, :_emits, :_parallelism
|
2
|
+
attr_accessor :_name, :_type, :_emits, :_parallelism, :_prepare
|
3
3
|
|
4
4
|
def name(v)
|
5
5
|
@_name = v
|
@@ -13,4 +13,8 @@ class Zillabyte::Harness::CommonNode
|
|
13
13
|
@_parallelism = v
|
14
14
|
end
|
15
15
|
|
16
|
+
def prepare(&block)
|
17
|
+
@_prepare = block
|
18
|
+
end
|
19
|
+
|
16
20
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require "zillabyte/harness/base"
|
2
|
+
require "socket"
|
2
3
|
|
3
4
|
class Zillabyte::Harness::Component < Zillabyte::Harness::Base
|
4
5
|
|
@@ -7,16 +8,16 @@ class Zillabyte::Harness::Component < Zillabyte::Harness::Base
|
|
7
8
|
Zillabyte::Harness::Helper.check_name("component", h._name, {})
|
8
9
|
h._options = Zillabyte::Harness::Helper.opt_parser()
|
9
10
|
if(h._options[:command] == :info)
|
10
|
-
h.
|
11
|
+
h._socket = TCPSocket.new(h._options[:host], h._options[:port])
|
11
12
|
info_hash = {"language" => "ruby", "name" => h._name, "flow_type" => "component", "multilang_version" => Zillabyte::VERSION}
|
12
|
-
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h.
|
13
|
+
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, h._socket)
|
13
14
|
end
|
14
15
|
h
|
15
16
|
end
|
16
17
|
|
17
18
|
def inputs(*args, &block)
|
18
19
|
op = Zillabyte::Harness::OperationHandler.new(self, Zillabyte::Harness::ComponentStream)
|
19
|
-
op.build_multilang_operation("
|
20
|
+
op.build_multilang_operation("component_input", *args, &block)
|
20
21
|
.add_operation_properties_to_info(:name, :type, :fields)
|
21
22
|
.handle_operation
|
22
23
|
.get_output_streams
|
@@ -1,11 +1,11 @@
|
|
1
|
-
class Zillabyte::Harness::
|
1
|
+
class Zillabyte::Harness::ComponentInput
|
2
2
|
attr_accessor :_app, :_node
|
3
3
|
|
4
4
|
class Node
|
5
5
|
attr_accessor :_name, :_type, :_emits, :_fields
|
6
6
|
|
7
7
|
def initialize()
|
8
|
-
@_type = "
|
8
|
+
@_type = "input"
|
9
9
|
@_emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
10
10
|
@_fields = []
|
11
11
|
end
|
@@ -1,11 +1,11 @@
|
|
1
|
-
class Zillabyte::Harness::
|
1
|
+
class Zillabyte::Harness::ComponentOutput
|
2
2
|
attr_accessor :_app, :_node
|
3
3
|
|
4
4
|
class Node
|
5
5
|
attr_accessor :_name, :_relation, :_type, :_columns, :_scope
|
6
6
|
|
7
7
|
def initialize()
|
8
|
-
@_type = '
|
8
|
+
@_type = 'output'
|
9
9
|
@_columns = []
|
10
10
|
end
|
11
11
|
|
@@ -4,7 +4,7 @@ class Zillabyte::Harness::ComponentStream < Zillabyte::Harness::Stream
|
|
4
4
|
|
5
5
|
def outputs(*args, &block)
|
6
6
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
7
|
-
op.build_multilang_operation("
|
7
|
+
op.build_multilang_operation("component_output", *args, &block)
|
8
8
|
.add_operation_properties_to_info(:name, :type, :columns, :scope)
|
9
9
|
.add_optional_operation_properties_to_info(:relation)
|
10
10
|
.create_arc_info_from_stream(self)
|
@@ -2,7 +2,7 @@ class Zillabyte::Harness::Each
|
|
2
2
|
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
4
|
class Node < Zillabyte::Harness::CommonNode
|
5
|
-
attr_accessor :_output_format, :
|
5
|
+
attr_accessor :_output_format, :_execute
|
6
6
|
|
7
7
|
def initialize()
|
8
8
|
@_name = "each_"+Zillabyte::Harness::Counter.get()
|
@@ -14,10 +14,6 @@ class Zillabyte::Harness::Each
|
|
14
14
|
@_output_format = v
|
15
15
|
end
|
16
16
|
|
17
|
-
def prepare(&block)
|
18
|
-
@_prepare = block
|
19
|
-
end
|
20
|
-
|
21
17
|
def execute(&block)
|
22
18
|
@_execute = block
|
23
19
|
end
|
@@ -49,8 +45,9 @@ class Zillabyte::Harness::Each
|
|
49
45
|
end
|
50
46
|
|
51
47
|
def run_operation
|
52
|
-
|
53
|
-
|
54
|
-
c.
|
48
|
+
host = @_app._options[:host]
|
49
|
+
port = @_app._options[:port]
|
50
|
+
c = Zillabyte::Harness::EachController.new(@_node, Zillabyte::Common::Progress.new)
|
51
|
+
c.run(host, port)
|
55
52
|
end
|
56
53
|
end
|
@@ -2,17 +2,13 @@ class Zillabyte::Harness::Filter
|
|
2
2
|
attr_accessor :_app, :_node, :_options
|
3
3
|
|
4
4
|
class Node < Zillabyte::Harness::CommonNode
|
5
|
-
attr_accessor :
|
5
|
+
attr_accessor :_keep
|
6
6
|
|
7
7
|
def initialize()
|
8
8
|
@_name = "filter_"+Zillabyte::Harness::Counter.get()
|
9
9
|
@_type = 'filter'
|
10
10
|
end
|
11
11
|
|
12
|
-
def prepare(&block)
|
13
|
-
@_prepare = block
|
14
|
-
end
|
15
|
-
|
16
12
|
def keep(&block)
|
17
13
|
@_keep = block
|
18
14
|
end
|
@@ -44,9 +40,10 @@ class Zillabyte::Harness::Filter
|
|
44
40
|
end
|
45
41
|
|
46
42
|
def run_operation
|
47
|
-
|
48
|
-
|
49
|
-
c.
|
43
|
+
host = @_app._options[:host]
|
44
|
+
port = @_app._options[:port]
|
45
|
+
c = Zillabyte::Harness::FilterController.new(@_node, Zillabyte::Common::Progress.new)
|
46
|
+
c.run(host, port)
|
50
47
|
end
|
51
48
|
|
52
49
|
end
|
@@ -47,8 +47,9 @@ class Zillabyte::Harness::GroupBy
|
|
47
47
|
end
|
48
48
|
|
49
49
|
def run_operation
|
50
|
-
|
50
|
+
host = @_app._options[:host]
|
51
|
+
port = @_app._options[:port]
|
51
52
|
c = Zillabyte::Harness::GroupByController.new(@_node, Zillabyte::Common::Progress.new)
|
52
|
-
c.run(
|
53
|
+
c.run(host, port)
|
53
54
|
end
|
54
55
|
end
|
@@ -5,6 +5,7 @@ class Zillabyte::Harness::Helper
|
|
5
5
|
META_NAMES = ["id", "confidence", "since", "source"]
|
6
6
|
ALLOWED_OUTPUT_FORMATS = [:replace, :merge]
|
7
7
|
ALLOWED_END_CYCLE_POLICIES = [:null_emit, :explicit, :infinite]
|
8
|
+
ALLOWED_TYPES = [:string, :integer, :float, :double, :boolean, :array, :map]
|
8
9
|
|
9
10
|
def self.opt_parser()
|
10
11
|
options = {
|
@@ -20,8 +21,11 @@ class Zillabyte::Harness::Helper
|
|
20
21
|
opts.on("--name NAME") do |v|
|
21
22
|
options[:name] = v
|
22
23
|
end
|
23
|
-
opts.on("--
|
24
|
-
options[:
|
24
|
+
opts.on("--host HOST") do |v|
|
25
|
+
options[:host] = v
|
26
|
+
end
|
27
|
+
opts.on("--port PORT") do |v|
|
28
|
+
options[:port] = v
|
25
29
|
end
|
26
30
|
opts.on("--file FNAME") do |v|
|
27
31
|
options[:file] = v
|
@@ -343,7 +347,7 @@ class Zillabyte::Harness::Helper
|
|
343
347
|
msg = "#{ee}Field data types must be SYMBOLS in \"#{relation_name}\". #{pp}"
|
344
348
|
Zillabyte::Harness::Helper.print_error(msg)
|
345
349
|
end
|
346
|
-
if(ctype
|
350
|
+
if(!ALLOWED_TYPES.member?(ctype))
|
347
351
|
msg = "#{ee}Invalid field data type at \"#{ctype}\" in \"#{relation_name}\". #{pp}"
|
348
352
|
Zillabyte::Harness::Helper.print_error(msg)
|
349
353
|
end
|
@@ -382,6 +386,24 @@ class Zillabyte::Harness::Helper
|
|
382
386
|
Zillabyte::Harness::Helper.check_field_format("outputs", pp, cname, ctype, node._name)
|
383
387
|
end
|
384
388
|
end
|
389
|
+
|
390
|
+
def self.check_loop_back(node_name, nodes)
|
391
|
+
ee = "Error in \"loop_back\": \n\t "
|
392
|
+
found = false
|
393
|
+
nodes.each do |node|
|
394
|
+
if node._name == node_name
|
395
|
+
found = true
|
396
|
+
if node._type == "source"
|
397
|
+
msg = "#{ee}Cannot loop back to source node \"#{node_name}\"!"
|
398
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
399
|
+
end
|
400
|
+
end
|
401
|
+
end
|
402
|
+
if !found
|
403
|
+
msg = "#{ee}The specified loop-back node \"#{node_name}\" was not found in the operations preceding it."
|
404
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
405
|
+
end
|
406
|
+
end
|
385
407
|
|
386
408
|
def self.get_non_option_args(args)
|
387
409
|
if args.nil?
|
@@ -422,7 +444,7 @@ class Zillabyte::Harness::Helper
|
|
422
444
|
- "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
423
445
|
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
424
446
|
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
425
|
-
- Field types must be SYMBOLS. The following types are allowed
|
447
|
+
- Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
|
426
448
|
OUTPUT
|
427
449
|
|
428
450
|
@@_print_check_source = <<-OUTPUT
|
@@ -530,7 +552,7 @@ OUTPUT
|
|
530
552
|
- "Inputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
531
553
|
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
532
554
|
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
533
|
-
- Field types must be SYMBOLS. The following types are allowed
|
555
|
+
- Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
|
534
556
|
OUTPUT
|
535
557
|
|
536
558
|
@@_print_check_component_sink = <<-OUTPUT
|
@@ -543,7 +565,7 @@ OUTPUT
|
|
543
565
|
- "Outputs" stream "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
544
566
|
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
545
567
|
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
546
|
-
- Field types must be SYMBOLS. The following types are allowed
|
568
|
+
- Field types must be SYMBOLS. The following types are allowed #{ALLOWED_TYPES.join(", ")}.
|
547
569
|
OUTPUT
|
548
570
|
|
549
571
|
@@_print_check_call_component = <<-OUTPUT
|
@@ -8,8 +8,8 @@ class Zillabyte::Harness::Join
|
|
8
8
|
@_name = "join_"+Zillabyte::Harness::Counter.get()
|
9
9
|
@_type = 'join'
|
10
10
|
@_join_type = :inner
|
11
|
-
@_lhs_stream = args[0]
|
12
|
-
@_rhs_stream = args[1]
|
11
|
+
@_lhs_stream = args[0]._name
|
12
|
+
@_rhs_stream = args[1]._name
|
13
13
|
options = args[2]
|
14
14
|
|
15
15
|
options.symbolize_keys!
|
@@ -18,24 +18,13 @@ module Zillabyte
|
|
18
18
|
DEATH_TIMEOUT = READ_SOFT_TIMEOUT * 6 # If no activity from the parent (even response to a ping) time before we kill ourself.
|
19
19
|
|
20
20
|
class << self
|
21
|
-
attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :
|
21
|
+
attr_accessor :mode, :emits, :end_cycle_policy, :pending_commands, :host, :port, :socket
|
22
22
|
end
|
23
23
|
|
24
24
|
self.pending_commands = []
|
25
25
|
|
26
26
|
def setup_pipes
|
27
|
-
|
28
|
-
if(pipe_name and File.exist?(pipe_name+".in"))
|
29
|
-
Storm::Protocol.pipe_to_java = File.open("#{pipe_name}.in","w+")
|
30
|
-
else
|
31
|
-
Storm::Protocol.pipe_to_java = $stdout
|
32
|
-
end
|
33
|
-
|
34
|
-
if(pipe_name and File.exist?(pipe_name+".out"))
|
35
|
-
Storm::Protocol.pipe_from_java = File.open("#{pipe_name}.out","r+")
|
36
|
-
else
|
37
|
-
Storm::Protocol.pipe_from_java = $stdin
|
38
|
-
end
|
27
|
+
Storm::Protocol.socket = TCPSocket.open(Storm::Protocol.host || "0.0.0.0", Storm::Protocol.port)
|
39
28
|
end
|
40
29
|
|
41
30
|
def read_message
|
@@ -46,7 +35,7 @@ module Zillabyte
|
|
46
35
|
loop do
|
47
36
|
|
48
37
|
# Read the next line.. The IO.select magic here is a way to minimize concurrency overhead
|
49
|
-
results = IO.select([Storm::Protocol.
|
38
|
+
results = IO.select([Storm::Protocol.socket], nil, nil, READ_SOFT_TIMEOUT)
|
50
39
|
|
51
40
|
# Process timeout...
|
52
41
|
if results.nil?
|
@@ -102,9 +91,8 @@ module Zillabyte
|
|
102
91
|
def send_msg_to_parent(msg)
|
103
92
|
# log msg
|
104
93
|
# log msg.to_json
|
105
|
-
Storm::Protocol.
|
106
|
-
Storm::Protocol.
|
107
|
-
Storm::Protocol.pipe_to_java.flush
|
94
|
+
Storm::Protocol.socket.write(msg.to_json+"\n")
|
95
|
+
Storm::Protocol.socket.write("end\n")
|
108
96
|
end
|
109
97
|
|
110
98
|
def send_pid(heartbeat_dir)
|
@@ -219,9 +207,10 @@ module Zillabyte
|
|
219
207
|
|
220
208
|
def ack(id); end
|
221
209
|
|
222
|
-
def run(
|
210
|
+
def run(host, port)
|
223
211
|
Storm::Protocol.mode = 'source'
|
224
|
-
Storm::Protocol.
|
212
|
+
Storm::Protocol.host = host
|
213
|
+
Storm::Protocol.port = port
|
225
214
|
setup_pipes
|
226
215
|
handshake
|
227
216
|
|
@@ -262,19 +251,24 @@ module Zillabyte
|
|
262
251
|
|
263
252
|
def execute(tuple); end
|
264
253
|
|
265
|
-
def run(
|
254
|
+
def run(host, port)
|
266
255
|
Storm::Protocol.mode = 'each'
|
267
|
-
Storm::Protocol.
|
256
|
+
Storm::Protocol.host = host
|
257
|
+
Storm::Protocol.port = port
|
268
258
|
setup_pipes
|
269
|
-
|
259
|
+
handshake
|
270
260
|
|
271
261
|
while true
|
272
262
|
begin
|
273
263
|
m = read_command()
|
274
264
|
if(m)
|
275
|
-
|
276
|
-
|
277
|
-
|
265
|
+
if m['command'] && m['command'] == 'prepare'
|
266
|
+
prepare()
|
267
|
+
else
|
268
|
+
t = Tuple.from_hash(m)
|
269
|
+
if(t)
|
270
|
+
execute t
|
271
|
+
end
|
278
272
|
end
|
279
273
|
end
|
280
274
|
rescue SignalException => e
|
@@ -297,19 +291,24 @@ module Zillabyte
|
|
297
291
|
|
298
292
|
def keep(tuple); end
|
299
293
|
|
300
|
-
def run(
|
294
|
+
def run(host, port)
|
301
295
|
Storm::Protocol.mode = 'filter'
|
302
|
-
Storm::Protocol.
|
296
|
+
Storm::Protocol.host = host
|
297
|
+
Storm::Protocol.port = port
|
303
298
|
setup_pipes
|
304
|
-
|
299
|
+
handshake
|
305
300
|
|
306
301
|
while true
|
307
302
|
begin
|
308
303
|
m = read_command()
|
309
304
|
if m
|
310
|
-
|
311
|
-
|
312
|
-
|
305
|
+
if m['command'] && m['command'] == 'prepare'
|
306
|
+
prepare()
|
307
|
+
else
|
308
|
+
t = Tuple.from_hash(m)
|
309
|
+
if t
|
310
|
+
emit(t) if keep t
|
311
|
+
end
|
313
312
|
end
|
314
313
|
end
|
315
314
|
rescue SignalException => e
|
@@ -326,9 +325,10 @@ module Zillabyte
|
|
326
325
|
class GroupBy
|
327
326
|
include Storm::Protocol
|
328
327
|
|
329
|
-
def run(
|
328
|
+
def run(host, port )
|
330
329
|
Storm::Protocol.mode = 'group_by'
|
331
|
-
Storm::Protocol.
|
330
|
+
Storm::Protocol.host = host
|
331
|
+
Storm::Protocol.port = port
|
332
332
|
setup_pipes
|
333
333
|
handshake
|
334
334
|
|
@@ -342,6 +342,8 @@ module Zillabyte
|
|
342
342
|
when 'aggregate'
|
343
343
|
t = Tuple.from_hash(msg)
|
344
344
|
aggregate t
|
345
|
+
when 'prepare'
|
346
|
+
prepare
|
345
347
|
when 'end_group'
|
346
348
|
end_group
|
347
349
|
end
|
@@ -432,6 +434,10 @@ module Zillabyte
|
|
432
434
|
def begin_group(*args)
|
433
435
|
self.instance_exec *args, &@harness._begin_group
|
434
436
|
end
|
437
|
+
|
438
|
+
def prepare(*args)
|
439
|
+
self.instance_exec *args, &@harness._prepare if @harness._prepare
|
440
|
+
end
|
435
441
|
|
436
442
|
def aggregate(*args)
|
437
443
|
self.instance_exec *args, &@harness._aggregate
|
@@ -30,10 +30,10 @@ class Zillabyte::Harness::OperationHandler
|
|
30
30
|
@_operation = Zillabyte::Harness::InjectedComponent.new(@_app, *args)
|
31
31
|
when "sink"
|
32
32
|
@_operation = Zillabyte::Harness::Sink.new(@_app, *args)
|
33
|
-
when "
|
34
|
-
@_operation = Zillabyte::Harness::
|
35
|
-
when "
|
36
|
-
@_operation = Zillabyte::Harness::
|
33
|
+
when "component_input"
|
34
|
+
@_operation = Zillabyte::Harness::ComponentInput.new(@_app, *args)
|
35
|
+
when "component_output"
|
36
|
+
@_operation = Zillabyte::Harness::ComponentOutput.new(@_app, *args)
|
37
37
|
else
|
38
38
|
throw "Unknown type"
|
39
39
|
end
|
@@ -93,13 +93,13 @@ class Zillabyte::Harness::OperationHandler
|
|
93
93
|
@_node_hash_optpars.each do |par|
|
94
94
|
node_hash[par] = node.instance_variable_get("@_#{par}") if node.instance_variable_defined?("@_#{par}")
|
95
95
|
end
|
96
|
-
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app.
|
96
|
+
Zillabyte::Harness::Helper.write_node_to_file(node_hash, @_app._socket)
|
97
97
|
|
98
98
|
# Create arc hashes from streams that feed into this node
|
99
99
|
@_arc_hash_streams.each do |stream, direction|
|
100
100
|
arc_hash = {"name" => stream._name, "origin" => stream._previous_node_name, "dest" => node._name}
|
101
101
|
arc_hash[direction] = 1 if !direction.nil?
|
102
|
-
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app.
|
102
|
+
Zillabyte::Harness::Helper.write_arc_to_file(arc_hash, @_app._socket)
|
103
103
|
end
|
104
104
|
elsif(@_app._options[:command] == :execute and @_app._options[:name] == node._name)
|
105
105
|
@_operation.run_operation
|
@@ -2,7 +2,7 @@ class Zillabyte::Harness::Source
|
|
2
2
|
attr_accessor :_app, :_node, :_relation
|
3
3
|
|
4
4
|
class Node < Zillabyte::Harness::CommonNode
|
5
|
-
attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
|
5
|
+
attr_accessor :_matches, :_relation, :_end_cycle_policy, :_begin_cycle, :_next_tuple
|
6
6
|
|
7
7
|
def initialize()
|
8
8
|
@_name = "source_"+Zillabyte::Harness::Counter.get()
|
@@ -35,9 +35,6 @@ class Zillabyte::Harness::Source
|
|
35
35
|
@_next_tuple = block
|
36
36
|
end
|
37
37
|
|
38
|
-
def prepare(&block)
|
39
|
-
@_prepare = block
|
40
|
-
end
|
41
38
|
end
|
42
39
|
|
43
40
|
def initialize(app, *args)
|
@@ -66,8 +63,9 @@ class Zillabyte::Harness::Source
|
|
66
63
|
end
|
67
64
|
|
68
65
|
def run_operation
|
69
|
-
|
70
|
-
|
71
|
-
c.
|
66
|
+
host = @_app._options[:host]
|
67
|
+
port = @_app._options[:port]
|
68
|
+
c = Zillabyte::Harness::SourceController.new(@_node, Zillabyte::Common::Progress.new)
|
69
|
+
c.run(host, port)
|
72
70
|
end
|
73
71
|
end
|
@@ -45,12 +45,13 @@ class Zillabyte::Harness::Stream
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def join_with(*args)
|
48
|
+
lhs_stream = self
|
49
|
+
rhs_stream = args[0]
|
48
50
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
49
|
-
op
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
.create_arc_info_from_stream(node._rhs_stream, :right)
|
51
|
+
op.build_multilang_operation("join", self, *args)
|
52
|
+
.add_operation_properties_to_info(:name, :type, :lhs_fields, :rhs_fields, :join_type, :lhs_stream, :rhs_stream)
|
53
|
+
.create_arc_info_from_stream(lhs_stream, :left)
|
54
|
+
.create_arc_info_from_stream(rhs_stream, :right)
|
54
55
|
.handle_operation
|
55
56
|
.get_output_streams
|
56
57
|
end
|
@@ -69,6 +70,14 @@ class Zillabyte::Harness::Stream
|
|
69
70
|
alias_method :executes, :call_component
|
70
71
|
alias_method :execute, :call_component
|
71
72
|
|
73
|
+
def loop_back(*args, &block)
|
74
|
+
# This is not a real operation, just telling the stream to loop back to the previous operation
|
75
|
+
loop_back_node = args[0]
|
76
|
+
Zillabyte::Harness::Helper.check_loop_back(loop_back_node, @_app._nodes)
|
77
|
+
Zillabyte::Harness::Helper.write_arc_to_file({"name" => self._name, "origin" => self._previous_node_name, "dest" => loop_back_node, "loop_back" => 1}, @_app._socket)
|
78
|
+
self
|
79
|
+
end
|
80
|
+
|
72
81
|
def sink(*args, &block)
|
73
82
|
op = Zillabyte::Harness::OperationHandler.new(@_app, self.class)
|
74
83
|
op.build_multilang_operation("sink", *args, &block)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-09-
|
11
|
+
date: 2014-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -24,34 +24,20 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: mkfifo
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - ">="
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: zillabyte-cli
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
44
30
|
requirements:
|
45
31
|
- - "~>"
|
46
32
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.
|
33
|
+
version: 0.9.0
|
48
34
|
type: :runtime
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|
52
38
|
- - "~>"
|
53
39
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.
|
40
|
+
version: 0.9.0
|
55
41
|
description: The Official Zillabyte Gem
|
56
42
|
email:
|
57
43
|
- gem@zillabyte.com
|
@@ -67,8 +53,8 @@ files:
|
|
67
53
|
- ruby/lib/zillabyte/harness/clump.rb
|
68
54
|
- ruby/lib/zillabyte/harness/common_node.rb
|
69
55
|
- ruby/lib/zillabyte/harness/component.rb
|
70
|
-
- ruby/lib/zillabyte/harness/
|
71
|
-
- ruby/lib/zillabyte/harness/
|
56
|
+
- ruby/lib/zillabyte/harness/component_input.rb
|
57
|
+
- ruby/lib/zillabyte/harness/component_output.rb
|
72
58
|
- ruby/lib/zillabyte/harness/component_stream.rb
|
73
59
|
- ruby/lib/zillabyte/harness/counter.rb
|
74
60
|
- ruby/lib/zillabyte/harness/each.rb
|