zillabyte 0.0.21 → 0.0.22
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/ruby/lib/zillabyte/harness/filter.rb +25 -0
- data/ruby/lib/zillabyte/harness/helper.rb +22 -0
- data/ruby/lib/zillabyte/harness/live_delegator.rb +50 -0
- data/ruby/lib/zillabyte/harness/sink.rb +5 -1
- data/ruby/lib/zillabyte/harness/stream.rb +24 -1
- data/ruby/lib/zillabyte/harness/tuple.rb +17 -4
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
ZGFiZGYyNGE4YTQzNzVhZWM1NTI1OTUzZDk4ZWU2ZGRmYWQxZmJhNw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
N2QzYWNhNmY4OTUyZWIwODU3NDBiYzA1ZDgyOGIxZTZlNmY2NTlmMQ==
|
7
7
|
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MTU1NDE2ZGNiODMwYjM2NzY4NDYyMzc1OTY4ZTA2NTg2MjIzNDNiNjk2MzY4
|
10
|
+
YjJkODU5ZTUwY2YzZTFmZjA3MDI1MGI4YmVhZmRhMDJkZWExNzBhMzhhNTg4
|
11
|
+
NmExNGViNTZkYWQxM2M2MDRiYzFjNjczNWQ2NmE5YjIzYmQ3YjU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NGYwODkxNGM3OTkxNGM4NTQwY2E4MzdiMjBjYzA0ZGM4YjA4ZTllNGIwYmIy
|
14
|
+
NmY4ZGEzNmYyZGZiN2FkNDU2YWQ0OTlmZWFjNzIyMDM1NDdiYTE2Mzk4M2M4
|
15
|
+
MjA0MTczNzEwY2Q0N2FlYTdjYjE2ZjllZWYwNWFmZGFjODg0MDc=
|
@@ -0,0 +1,25 @@
|
|
1
|
+
class Zillabyte::Harness::Filter
|
2
|
+
attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_keep
|
3
|
+
|
4
|
+
def initialize()
|
5
|
+
@_name = "filter_"+Zillabyte::Harness::Counter.get()
|
6
|
+
@_type = 'filter'
|
7
|
+
end
|
8
|
+
|
9
|
+
def name(v)
|
10
|
+
@_name = v
|
11
|
+
end
|
12
|
+
|
13
|
+
def emits(*v)
|
14
|
+
@_emits = *v
|
15
|
+
end
|
16
|
+
|
17
|
+
def prepare(&block)
|
18
|
+
@_prepare = block
|
19
|
+
end
|
20
|
+
|
21
|
+
def keep(&block)
|
22
|
+
@_keep = block
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
@@ -120,6 +120,16 @@ class Zillabyte::Harness::Helper
|
|
120
120
|
end
|
121
121
|
end
|
122
122
|
|
123
|
+
def self.check_filter(filter)
|
124
|
+
ee = "Error in \"filter\": \n\t "
|
125
|
+
pp = @@_print_check_filter
|
126
|
+
|
127
|
+
if !filter._keep
|
128
|
+
msg = "#{ee}A \"filter\" must contain a \"condition\" block. #{pp}"
|
129
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
130
|
+
end
|
131
|
+
end
|
132
|
+
|
123
133
|
def self.check_each(each)
|
124
134
|
ee = "Error in \"each\": \n\t "
|
125
135
|
pp = @@_print_check_each
|
@@ -232,6 +242,18 @@ OUTPUT
|
|
232
242
|
* :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
|
233
243
|
OUTPUT
|
234
244
|
|
245
|
+
@@_print_check_filter = <<-OUTPUT
|
246
|
+
\n\n"Filter" Syntax:
|
247
|
+
Simplified syntax:
|
248
|
+
stream.filter { |tuple| |=condition=| }
|
249
|
+
|
250
|
+
* Outputs a given tuple from the stream only if condition is true
|
251
|
+
Custom filter:
|
252
|
+
stream.filter do |tuple|
|
253
|
+
|=condition=|
|
254
|
+
end
|
255
|
+
OUTPUT
|
256
|
+
|
235
257
|
@@_print_check_each = <<-OUTPUT
|
236
258
|
\n\n"Each" Syntax:
|
237
259
|
Simplified syntax:
|
@@ -283,6 +283,35 @@ module Zillabyte
|
|
283
283
|
end
|
284
284
|
end
|
285
285
|
|
286
|
+
class Filter
|
287
|
+
include Storm::Protocol
|
288
|
+
|
289
|
+
def prepare(conf, context); end
|
290
|
+
|
291
|
+
def keep(tuple); end
|
292
|
+
|
293
|
+
def run(pipe_name)
|
294
|
+
Storm::Protocol.mode = 'filter'
|
295
|
+
Storm::Protocol.pipe_name = pipe_name
|
296
|
+
setup_pipes
|
297
|
+
prepare(*handshake)
|
298
|
+
|
299
|
+
while true
|
300
|
+
begin
|
301
|
+
t = Tuple.from_hash(read_command)
|
302
|
+
next if(!t)
|
303
|
+
keep t
|
304
|
+
rescue SignalException => e
|
305
|
+
raise
|
306
|
+
rescue Exception => e
|
307
|
+
fail 'Exception in filter: ' + e.message + ' - ' + e.backtrace.join('\n')
|
308
|
+
# We may recover from this, but let JVM decide
|
309
|
+
end
|
310
|
+
done
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
286
315
|
class GroupBy
|
287
316
|
include Storm::Protocol
|
288
317
|
|
@@ -358,6 +387,27 @@ module Zillabyte
|
|
358
387
|
|
359
388
|
end
|
360
389
|
|
390
|
+
class FilterController < Storm::Filter
|
391
|
+
|
392
|
+
def initialize(harness, progress)
|
393
|
+
@harness = harness
|
394
|
+
@progress = progress
|
395
|
+
Storm::Protocol.emits = harness._emits
|
396
|
+
end
|
397
|
+
|
398
|
+
def keep(*args)
|
399
|
+
if self.instance_exec *args, &@harness._keep
|
400
|
+
emit args[0]
|
401
|
+
end
|
402
|
+
end
|
403
|
+
|
404
|
+
def prepare(*args)
|
405
|
+
self.instance_exec(*args, &@harness._prepare) if @harness._prepare
|
406
|
+
end
|
407
|
+
|
408
|
+
|
409
|
+
end
|
410
|
+
|
361
411
|
class GroupByController < Storm::GroupBy
|
362
412
|
|
363
413
|
def initialize(harness, progress)
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::Sink
|
2
|
-
attr_accessor :_name, :_relation, :_type, :_columns, :_consumes
|
2
|
+
attr_accessor :_name, :_relation, :_type, :_columns, :_consumes, :_scope
|
3
3
|
|
4
4
|
def initialize()
|
5
5
|
@_type = 'sink'
|
@@ -24,4 +24,8 @@ class Zillabyte::Harness::Sink
|
|
24
24
|
@_columns << col
|
25
25
|
end
|
26
26
|
|
27
|
+
def scope(v)
|
28
|
+
@_scope = v
|
29
|
+
end
|
30
|
+
|
27
31
|
end
|
@@ -49,6 +49,29 @@ class Zillabyte::Harness::Stream
|
|
49
49
|
output_streams
|
50
50
|
end
|
51
51
|
|
52
|
+
def filter(&block)
|
53
|
+
h = Zillabyte::Harness::Filter.new()
|
54
|
+
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
55
|
+
h._keep = block
|
56
|
+
|
57
|
+
@_app._nodes << h
|
58
|
+
if(@_app._options[:command] == :info)
|
59
|
+
info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
|
60
|
+
info_hash["emits"] = h._emits
|
61
|
+
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
|
62
|
+
elsif(@_app._options[:command] == :execute and @_app._options[:name] == h._name)
|
63
|
+
pipe_name = @_app._options[:pipe]
|
64
|
+
c = Zillabyte::Harness::FilterController.new(h, progress = Zillabyte::Common::Progress.new)
|
65
|
+
c.run(pipe_name)
|
66
|
+
end
|
67
|
+
|
68
|
+
output_streams = []
|
69
|
+
h._emits.each do |stream|
|
70
|
+
output_streams << Zillabyte::Harness::Stream.new(stream, @_app, h._name)
|
71
|
+
end
|
72
|
+
output_streams = output_streams[0] if output_streams.size == 1
|
73
|
+
output_streams
|
74
|
+
end
|
52
75
|
|
53
76
|
def group_by(*args, &block)
|
54
77
|
|
@@ -164,7 +187,7 @@ class Zillabyte::Harness::Stream
|
|
164
187
|
Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
|
165
188
|
@_app._nodes << h
|
166
189
|
if(@_app._options[:command] == :info)
|
167
|
-
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name}
|
190
|
+
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name, "scope" => h._scope}
|
168
191
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_app._info_file)
|
169
192
|
end
|
170
193
|
end
|
@@ -6,29 +6,42 @@ class Tuple
|
|
6
6
|
end
|
7
7
|
|
8
8
|
def [](name)
|
9
|
+
name = name.to_s unless name.is_a?(String)
|
9
10
|
maybe_build_alias_hash()
|
10
|
-
key = (@_alias_hash[name] || name)
|
11
|
+
key = (@_alias_hash[name] || name)
|
11
12
|
if values.has_key?( key )
|
12
13
|
return values[key]
|
13
14
|
else
|
14
|
-
raise ArgumentError, "The requested tuple does not contain the key: \"#{name}\". Current
|
15
|
+
raise ArgumentError, "The requested tuple does not contain the key: \"#{name}\". Current tuple contains keys: #{values.keys.to_s} with aliases #{@_alias_hash.to_s}"
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
18
19
|
def values
|
19
20
|
@hash["tuple"]
|
20
21
|
end
|
22
|
+
|
23
|
+
def aliased_hash
|
24
|
+
out = {}
|
25
|
+
column_aliases.each do |col_hash|
|
26
|
+
concrete = col_hash["concrete_name"]
|
27
|
+
aliased = col_hash["alias"]
|
28
|
+
out[aliased] = values[concrete]
|
29
|
+
end
|
30
|
+
return out
|
31
|
+
end
|
32
|
+
|
33
|
+
def keys
|
34
|
+
@hash.keys
|
35
|
+
end
|
21
36
|
|
22
37
|
def meta
|
23
38
|
@hash["meta"]
|
24
39
|
end
|
25
40
|
|
26
|
-
|
27
41
|
def to_s
|
28
42
|
values.to_s
|
29
43
|
end
|
30
44
|
|
31
|
-
|
32
45
|
def column_aliases
|
33
46
|
@hash['column_aliases'] || []
|
34
47
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.22
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.0.
|
47
|
+
version: 0.0.22
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.0.
|
54
|
+
version: 0.0.22
|
55
55
|
description: The Official Zillabyte Gem
|
56
56
|
email:
|
57
57
|
- gem@zillabyte.com
|
@@ -63,6 +63,7 @@ files:
|
|
63
63
|
- ruby/lib/zillabyte/harness/app.rb
|
64
64
|
- ruby/lib/zillabyte/harness/counter.rb
|
65
65
|
- ruby/lib/zillabyte/harness/each.rb
|
66
|
+
- ruby/lib/zillabyte/harness/filter.rb
|
66
67
|
- ruby/lib/zillabyte/harness/group_by.rb
|
67
68
|
- ruby/lib/zillabyte/harness/helper.rb
|
68
69
|
- ruby/lib/zillabyte/harness/join.rb
|
@@ -94,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
94
95
|
version: '0'
|
95
96
|
requirements: []
|
96
97
|
rubyforge_project:
|
97
|
-
rubygems_version: 2.0.
|
98
|
+
rubygems_version: 2.0.7
|
98
99
|
signing_key:
|
99
100
|
specification_version: 4
|
100
101
|
summary: The Official Zillabyte Gem
|