zillabyte 0.0.17 → 0.0.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +9 -9
- data/ruby/lib/zillabyte/harness/app.rb +11 -5
- data/ruby/lib/zillabyte/harness/each.rb +1 -5
- data/ruby/lib/zillabyte/harness/helper.rb +95 -214
- data/ruby/lib/zillabyte/harness/source.rb +1 -4
- data/ruby/lib/zillabyte/harness/stream.rb +8 -5
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +5 -6
- data/ruby/README.md +0 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YWVjZWZiYTQwODcyODk2OGE3MzM1NWFiMzkwZmVkM2E3NzgyZWFkZg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
7
|
-
|
6
|
+
MTczN2E4NzUzMjA2ZWFiMjFhZjJmODNiNGE4ZWRhNTViZjhmOGVmNQ==
|
7
|
+
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NWM3YmZkOWIyYTNiNzhhYmViMmRkMTkwY2E5NjhmMDA4NDEyMmVjYmNhNDk1
|
10
|
+
ZDQ4MmJmNTFjNjQxMzAyZjIzOGQ5YmM0NjgyMzhlNWMyMTMwZWFkNTA3NmVk
|
11
|
+
N2QyZjFhYWU5YTViZjYwN2UzYmVkMDU0YzEyYzAzMDNkYzI0ZGM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YzgxYjAwOTUzMTNiMWI5MGJkMjE0ZTViZGU2MTc2NDRlNDJhZGIyYjJiNTBj
|
14
|
+
NzhjOGYxMTE1ZTRjMjVmZGExZmE2MmM4OGJmOWNhYTFiZTkxZDg3NzQ5ZGVk
|
15
|
+
ODg4NmRhZmZjYjE5OGExZTUxYWI4ZWViZTE5NWM4ZWQ3Y2RhMDU=
|
@@ -1,12 +1,13 @@
|
|
1
1
|
require "json"
|
2
2
|
|
3
3
|
class Zillabyte::Harness::App
|
4
|
-
attr_accessor :_name, :_options, :_nodes, :_streams, :_info_file
|
4
|
+
attr_accessor :_name, :_options, :_nodes, :_streams, :_names, :_info_file
|
5
5
|
|
6
6
|
def self.build(name)
|
7
7
|
h = Zillabyte::Harness::App.new()
|
8
8
|
h._nodes = []
|
9
9
|
h._streams = {}
|
10
|
+
h._names = {}
|
10
11
|
h._name = name
|
11
12
|
Zillabyte::Harness::Helper.check_name("new", h._name, {})
|
12
13
|
h._options = Zillabyte::Harness::Helper.opt_parser()
|
@@ -23,23 +24,28 @@ class Zillabyte::Harness::App
|
|
23
24
|
# Are we given a block?
|
24
25
|
if block_given?
|
25
26
|
h.instance_eval(&block)
|
26
|
-
Zillabyte::Harness::Helper.check_name("source", h._name,
|
27
|
-
Zillabyte::Harness::Helper.
|
28
|
-
if
|
27
|
+
Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
|
28
|
+
Zillabyte::Harness::Helper.check_custom_source(h)
|
29
|
+
if h._emits
|
30
|
+
Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams)
|
31
|
+
else
|
29
32
|
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
30
33
|
end
|
31
34
|
# No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
|
32
35
|
else
|
33
36
|
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
37
|
+
Zillabyte::Harness::Helper.check_source_args(args)
|
34
38
|
h.matches(args[0])
|
35
39
|
end
|
36
40
|
@_nodes << h
|
37
41
|
if(@_options[:command] == :info)
|
38
|
-
info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits
|
42
|
+
info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
|
39
43
|
if(h._relation)
|
40
44
|
info_hash["relation"] = h._relation
|
41
45
|
elsif(h._matches)
|
42
46
|
info_hash["matches"] = h._matches
|
47
|
+
else
|
48
|
+
info_hash["end_cycle_policy"] = h._end_cycle_policy
|
43
49
|
end
|
44
50
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
|
45
51
|
elsif(@_options[:command] == :execute and @_options[:name] == h._name)
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::Each
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
|
3
3
|
|
4
4
|
def initialize()
|
5
5
|
@_name = "each_"+Zillabyte::Harness::Counter.get()
|
@@ -14,10 +14,6 @@ class Zillabyte::Harness::Each
|
|
14
14
|
@_emits = *v
|
15
15
|
end
|
16
16
|
|
17
|
-
def emit(*v)
|
18
|
-
@_has_emit = true
|
19
|
-
end
|
20
|
-
|
21
17
|
def prepare(&block)
|
22
18
|
@_prepare = block
|
23
19
|
end
|
@@ -35,6 +35,10 @@ class Zillabyte::Harness::Helper
|
|
35
35
|
exit(1)
|
36
36
|
end
|
37
37
|
|
38
|
+
def self.print_warning(msg)
|
39
|
+
STDERR.write(msg+"\n")
|
40
|
+
end
|
41
|
+
|
38
42
|
def self.check_name(operation, name, names)
|
39
43
|
ee = "Error in \"#{operation}\" at \"name\": \n\t "
|
40
44
|
|
@@ -51,92 +55,87 @@ class Zillabyte::Harness::Helper
|
|
51
55
|
|
52
56
|
end
|
53
57
|
|
54
|
-
def self.
|
55
|
-
ee = "Error in \"
|
58
|
+
def self.check_custom_source(source)
|
59
|
+
ee = "Error in \"source\": \n\t "
|
56
60
|
pp = @@_print_check_source
|
57
61
|
|
58
62
|
rm = !source._relation.nil?
|
59
63
|
mm = !source._matches.nil?
|
60
|
-
if( rm
|
64
|
+
if( rm or mm )
|
61
65
|
msg = "#{ee}"
|
62
|
-
msg += "A \"
|
66
|
+
msg += "A custom \"source\" may not contain a \"matches\" clause. To source from a relation, see the following. #{pp}"
|
63
67
|
Zillabyte::Harness::Helper.print_error(msg)
|
64
68
|
end
|
69
|
+
|
65
70
|
if(rm or mm)
|
66
|
-
if source.
|
67
|
-
msg = "#{ee}If \"
|
71
|
+
if source._begin_cycle or source._next_tuple
|
72
|
+
msg = "#{ee}If \"source\" has a \"matches\\relation\" clause, it may not have a \"begin_cycle\" or \"next_tuple\" block. #{pp}"
|
68
73
|
Zillabyte::Harness::Helper.print_error(msg)
|
69
74
|
end
|
70
75
|
end
|
76
|
+
|
77
|
+
allowed_policies = [:null_emit, :explicit]
|
78
|
+
if !(allowed_policies.member? source._end_cycle_policy)
|
79
|
+
msg = "#{ee}Invalid \"end_cycle_policy\": \"#{source._end_cycle_policy}\". The allowed policies are :null_emit and :explicit. #{pp}"
|
80
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
81
|
+
end
|
82
|
+
|
83
|
+
if !source._next_tuple
|
84
|
+
msg = "#{ee}A custom \"source\" must contain a \"next_tuple\" block. #{pp}"
|
85
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.check_source_args(args)
|
90
|
+
ee = "Error in \"source\": \n\t "
|
91
|
+
pp = @@_print_check_source
|
92
|
+
|
93
|
+
if(args.length != 1)
|
94
|
+
msg = "#{ee}Invalid number of arguments to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
|
95
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
96
|
+
end
|
97
|
+
|
98
|
+
if(!args[0].instance_of?(String) and !args[0].instance_of?(Array))
|
99
|
+
msg = "#{ee}Invalid argument to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
|
100
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
101
|
+
end
|
71
102
|
end
|
72
103
|
|
73
104
|
def self.check_emits(operation, emits, streams)
|
74
105
|
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
75
|
-
pp =
|
76
|
-
nn = "stream"
|
106
|
+
pp = ""
|
77
107
|
|
78
108
|
if(!emits.instance_of?(Array))
|
79
|
-
msg = "#{ee}\"Emits\" must be a sequence of
|
109
|
+
msg = "#{ee}\"Emits\" must be a sequence of comma-separated STRINGS. #{pp}"
|
80
110
|
Zillabyte::Harness::Helper.print_error(msg)
|
81
111
|
end
|
82
112
|
n_emits = emits.length
|
83
113
|
if(n_emits == 0)
|
84
|
-
msg = "#{ee}Must emit at least one
|
114
|
+
msg = "#{ee}Must emit at least one stream, \"emits\" cannot be an empty. #{pp}"
|
85
115
|
Zillabyte::Harness::Helper.print_error(msg)
|
86
116
|
end
|
87
117
|
|
88
|
-
current_op_streams = {}
|
89
118
|
emits.each do |e|
|
90
119
|
if(!e.instance_of?(String) or (e =~ /^\w+$/).nil?)
|
91
|
-
msg = "#{ee}\"Emits\"
|
120
|
+
msg = "#{ee}\"Emits\" stream name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
|
92
121
|
Zillabyte::Harness::Helper.print_error(msg)
|
93
122
|
end
|
94
|
-
if(
|
95
|
-
msg = "#{ee}The
|
123
|
+
if(streams[e])
|
124
|
+
msg = "#{ee}The stream name \"#{e}\" is previously defined! #{pp}"
|
96
125
|
Zillabyte::Harness::Helper.print_error(msg)
|
97
126
|
end
|
98
|
-
|
99
|
-
end
|
100
|
-
if(n_emits > 1)
|
101
|
-
return true
|
102
|
-
else
|
103
|
-
return false
|
127
|
+
streams[e] = operation
|
104
128
|
end
|
105
|
-
|
106
129
|
end
|
107
130
|
|
108
|
-
def self.
|
109
|
-
ee = "Error in \"
|
110
|
-
pp =
|
111
|
-
if(operation == "aggregate")
|
112
|
-
pp = @@_print_check_group_by
|
113
|
-
end
|
131
|
+
def self.check_each(each)
|
132
|
+
ee = "Error in \"each\": \n\t "
|
133
|
+
pp = @@_print_check_each
|
114
134
|
|
115
|
-
|
116
|
-
|
117
|
-
msg = "#{ee}Group_by must be an ARRAY with at least one element! #{pp}"
|
135
|
+
if !each._execute
|
136
|
+
msg = "#{ee}An \"each\" must contain an \"execute\" block. #{pp}"
|
118
137
|
Zillabyte::Harness::Helper.print_error(msg)
|
119
|
-
end
|
120
|
-
|
121
|
-
if(operation == "aggregate")
|
122
|
-
# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from source!
|
123
|
-
group_by.each do |f|
|
124
|
-
if(h._consumes)
|
125
|
-
fields = streams[h._consumes]
|
126
|
-
if(!fields.include?(f))
|
127
|
-
msg = "#{ee}Group_by field not in consumed stream \"#{h._consumes}\". #{pp}"
|
128
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
129
|
-
end
|
130
|
-
else
|
131
|
-
previous_emits = nodes.last._emits[0] #if no "consumes", then operation consumes the sole stream from the previous operation
|
132
|
-
if(!previous_emits[1].include?(f))
|
133
|
-
msg = "#{ee}Group_by field not in \"emits\" of previous relation \"#{previous_emits[0]}\". #{pp}"
|
134
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
138
|
+
end
|
140
139
|
end
|
141
140
|
|
142
141
|
def self.check_sink(sink, nodes)
|
@@ -174,18 +173,13 @@ class Zillabyte::Harness::Helper
|
|
174
173
|
columns.each do |col|
|
175
174
|
cname = col.keys()[0]
|
176
175
|
ctype = col[cname]
|
177
|
-
Zillabyte::Harness::Helper.check_sink_column_format(
|
176
|
+
Zillabyte::Harness::Helper.check_sink_column_format(cname,ctype,name)
|
178
177
|
end
|
179
178
|
end
|
180
179
|
|
181
|
-
def self.check_sink_column_format(
|
182
|
-
|
183
|
-
|
184
|
-
pp = @@_print_check_sink
|
185
|
-
elsif(operation.include?("simple"))
|
186
|
-
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
187
|
-
pp = @@_print_check_simple_app_emits
|
188
|
-
end
|
180
|
+
def self.check_sink_column_format(cname, ctype, relation_name)
|
181
|
+
ee = "Error in \"sink\" at \"column\": \n\t "
|
182
|
+
pp = @@_print_check_sink
|
189
183
|
|
190
184
|
if(!cname.instance_of?(String) or (cname =~ /^\w+$/).nil?)
|
191
185
|
msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in relation \"#{relation_name}\". #{pp}"
|
@@ -205,60 +199,6 @@ class Zillabyte::Harness::Helper
|
|
205
199
|
end
|
206
200
|
end
|
207
201
|
|
208
|
-
def self.check_sink_consumes(sink, streams)
|
209
|
-
ee = "Error in \"sink\" at \"consumes\": \n\t "
|
210
|
-
pp = @@_print_check_sink
|
211
|
-
|
212
|
-
name = sink._name
|
213
|
-
columns = sink._columns
|
214
|
-
consumes = sink._consumes
|
215
|
-
|
216
|
-
stream_fields = streams[consumes]
|
217
|
-
if(stream_fields.length != columns.length)
|
218
|
-
msg = "#{ee}Number of columns in \"sink\" differs from number of fields in the consumed stream at relation \"#{name}\". #{pp}"
|
219
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
220
|
-
end
|
221
|
-
columns.each do |col|
|
222
|
-
col_name = col.keys()[0]
|
223
|
-
if(!stream_fields.include?(col_name))
|
224
|
-
msg = "#{ee}The column \"#{col_name}\", is not emitted by the stream \"#{consumes}\". #{pp}"
|
225
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
226
|
-
end
|
227
|
-
end
|
228
|
-
end
|
229
|
-
|
230
|
-
def self.check_simple_app_emits(operation, emits)
|
231
|
-
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
232
|
-
pp = @@_print_check_simple_app_emits
|
233
|
-
|
234
|
-
name = emits[0]
|
235
|
-
columns = emits[1]
|
236
|
-
if(!columns.instance_of?(Array))
|
237
|
-
msg = "#{ee}Field names must be an ARRAY of HASHES in relation \"#{name}\". #{pp}"
|
238
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
239
|
-
end
|
240
|
-
columns.each do |col|
|
241
|
-
if(!col.instance_of?(Hash))
|
242
|
-
msg = "#{ee}Fields names must be listed in HASH format in relation \"#{name}\". #{pp}"
|
243
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
244
|
-
end
|
245
|
-
colkeys = col.keys()
|
246
|
-
if(colkeys.length != 1)
|
247
|
-
msg = "#{ee}Each field must be a separate HASH with {field_name : data_type} in relation \"#{name}\". #{pp}"
|
248
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
249
|
-
end
|
250
|
-
colkey = colkeys[0]
|
251
|
-
colval = col[colkey]
|
252
|
-
Zillabyte::Harness::Helper.check_sink_column_format(operation,colkey,colval,name)
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
def self.print_simple_app_usage(msg)
|
257
|
-
ee = "Error in \"simple_app\": \n\t "
|
258
|
-
msg = "#{ee}#{msg} #{@@_print_check_simple_app} #{@@_print_check_simple_app_emits}"
|
259
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
260
|
-
end
|
261
|
-
|
262
202
|
# Test helper...
|
263
203
|
def self.argv()
|
264
204
|
@_argv || ARGV
|
@@ -269,100 +209,61 @@ class Zillabyte::Harness::Helper
|
|
269
209
|
@_argv = v
|
270
210
|
end
|
271
211
|
|
272
|
-
@@_print_check_emits = <<-OUTPUT
|
273
|
-
"Emits" Syntax:
|
274
|
-
- "Emits" must be a non-empty ARRAY.
|
275
|
-
- Each element of "emits" must be an ARRAY of length = 2.
|
276
|
-
* The first element should be the unique stream name defined as a non-empty STRING with only alphanumeric and underscore characters.
|
277
|
-
* The second element should be an ARRAY of field names for that stream.
|
278
|
-
e.g.
|
279
|
-
emits = [ [ "stream_1", [ "field_11", "field_12", ... ] ],
|
280
|
-
[ "stream_2", [ "field_21", "field_22", ... ] ] ] .
|
281
|
-
- Stream and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
|
282
|
-
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
283
|
-
OUTPUT
|
284
|
-
|
285
|
-
|
286
|
-
@@_print_check_group_by =<<-OUTPUT
|
287
|
-
"Group_by" Syntax:
|
288
|
-
- "Group_by" must be a non-empty ARRAY.
|
289
|
-
- Each element of "group_by" must be a STRING corresponding to a field emitted by the previous operation or the stream
|
290
|
-
consumed by the aggregation operation.
|
291
|
-
OUTPUT
|
292
|
-
|
293
|
-
@@_print_check_simple_app_emits = <<-OUTPUT
|
294
|
-
"Emits" Syntax:
|
295
|
-
- "Emits" must be a non-empty ARRAY.
|
296
|
-
- Each element of "emits" must be an ARRAY of length = 2.
|
297
|
-
* The first element should be the unique relation name defined as a non-empty STRING with only alphanumeric and underscore characters.
|
298
|
-
* The second element should be an ARRAY of HASHES with field names and data types for that relation. e.g.
|
299
|
-
emits = [ [ "relation_1", [ {"field_11" => "type_11"}, {"field_12" => "type_12"}, ... ] ],
|
300
|
-
[ "relation_2", [ {"field_21" => "type_21"}, {"field_22" => "type_22"}, ... ] ] ] .
|
301
|
-
- Relation and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
|
302
|
-
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
303
|
-
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
304
|
-
OUTPUT
|
305
|
-
|
306
212
|
@@_print_check_sink = <<-OUTPUT
|
307
|
-
"Sink" Syntax:
|
308
|
-
|
309
|
-
Single stream:
|
310
|
-
app.sink do
|
213
|
+
\n\n"Sink" Syntax:
|
214
|
+
stream.sink do
|
311
215
|
name "name_of_relation"
|
312
|
-
column "field_1" :type_1
|
313
|
-
column "field_2" :type_2 ...
|
314
|
-
end
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
column "field_2" :type_2 ...
|
321
|
-
end
|
322
|
-
- "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
323
|
-
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
324
|
-
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
325
|
-
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
326
|
-
- If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
|
216
|
+
column "field_1", :type_1
|
217
|
+
column "field_2", :type_2 ...
|
218
|
+
end
|
219
|
+
- "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
220
|
+
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
221
|
+
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
222
|
+
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
223
|
+
- If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
|
327
224
|
* "Consumes" is the name of a stream emitted by an "each" or a "source" which the "sink" should save as a table.
|
328
225
|
* The columns specified in "sink" must match the fields emitted by the stream."
|
329
226
|
OUTPUT
|
330
227
|
|
331
228
|
@@_print_check_source = <<-OUTPUT
|
332
|
-
"Source" Syntax:
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
matches "sql_query or sxp"
|
338
|
-
end
|
339
|
-
Custom source:
|
229
|
+
\n\n"Source" Syntax:
|
230
|
+
Sourcing from a relation:
|
231
|
+
app.source("SQL query" or [SXP queries])
|
232
|
+
|
233
|
+
Custom source:
|
340
234
|
app.source do
|
341
|
-
name "name"
|
342
|
-
emits
|
343
|
-
|
344
|
-
|
345
|
-
|
235
|
+
name "name" \t\t\t\t\t => optional
|
236
|
+
emits "stream_1", "stream_2", ... \t\t => optional for single output stream
|
237
|
+
end_cycle_policy :null_emit OR :explicit \t => default :null_emit
|
238
|
+
begin_cycle |=block=| \t\t\t\t => optional if no initialization needed
|
239
|
+
next_tuple |=block=|
|
240
|
+
end
|
241
|
+
- The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
|
242
|
+
* :null_emit - end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
|
243
|
+
* :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
|
346
244
|
OUTPUT
|
347
245
|
|
348
246
|
@@_print_check_each = <<-OUTPUT
|
349
|
-
"Each" Syntax:
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
247
|
+
\n\n"Each" Syntax:
|
248
|
+
Simplified syntax:
|
249
|
+
stream.each do |tuple|
|
250
|
+
|=block=|
|
251
|
+
end
|
252
|
+
- This is equivalent to just specifying an \"execute\" block below.
|
253
|
+
- Note that this syntax only works for a single output stream.
|
254
|
+
|
255
|
+
Custom each:
|
256
|
+
stream.each do
|
257
|
+
name "name" \t\t\t\t\t => optional
|
258
|
+
emits "stream_1", "stream_2", ... \t\t => optional for single output stream
|
259
|
+
prepare |=block=| \t\t\t\t => optional if no initialization needed
|
260
|
+
execute |=block=|
|
357
261
|
end
|
358
|
-
- If there are multiple streams, "consumes" must be specified as a non-empty STRING!
|
359
|
-
* "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "each" operates on."
|
360
262
|
OUTPUT
|
361
263
|
|
362
264
|
@@_print_check_aggregate = <<-OUTPUT
|
363
|
-
"Aggregate" Syntax:
|
364
|
-
|
365
|
-
app.aggregate do
|
265
|
+
\n\n"Aggregate" Syntax:
|
266
|
+
stream.aggregate do
|
366
267
|
name "name" => optional
|
367
268
|
group_by ["field_1", "field_2", ...]
|
368
269
|
emits emits
|
@@ -371,29 +272,9 @@ OUTPUT
|
|
371
272
|
aggregate ...
|
372
273
|
end_group ...
|
373
274
|
end
|
374
|
-
|
275
|
+
- If there are multiple streams, "consumes" must be specified as a non-empty STRING!
|
375
276
|
* "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "aggregate" operates on.
|
376
277
|
OUTPUT
|
377
278
|
|
378
|
-
@@_print_check_simple_app = <<-OUTPUT
|
379
|
-
"Simple_app" Syntax:
|
380
|
-
- Simple_apps must be specified using the following syntax:
|
381
|
-
Sinking directly from a custom source:
|
382
|
-
Zillabyte.simple_app do
|
383
|
-
name "name"
|
384
|
-
emits emits
|
385
|
-
prepare ...
|
386
|
-
next_batch ...
|
387
|
-
end
|
388
|
-
Processing and sinking data from a relation:
|
389
|
-
Zillabyte.simple_app do
|
390
|
-
name "name"
|
391
|
-
matches "sql_query or sxp"
|
392
|
-
emits emits
|
393
|
-
prepare ...
|
394
|
-
execute ...
|
395
|
-
end
|
396
|
-
OUTPUT
|
397
|
-
|
398
279
|
@@_meta_names = ["id", "confidence", "since", "source"]
|
399
280
|
end
|
@@ -14,14 +14,18 @@ class Zillabyte::Harness::Stream
|
|
14
14
|
# Does the block take 0 arguments? If so it's not just an execute block.
|
15
15
|
if(block.arity == 0)
|
16
16
|
h.instance_eval(&block)
|
17
|
+
Zillabyte::Harness::Helper.check_name("each", h._name, @_app._names)
|
18
|
+
Zillabyte::Harness::Helper.check_each(h)
|
19
|
+
if h._emits
|
20
|
+
Zillabyte::Harness::Helper.check_emits("each", h._emits, @_app._streams)
|
21
|
+
else
|
22
|
+
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
23
|
+
end
|
17
24
|
# Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
|
18
25
|
else
|
19
26
|
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
20
|
-
h._has_emit = true
|
21
27
|
h._execute = block
|
22
28
|
end
|
23
|
-
Zillabyte::Harness::Helper.check_name("each", h._name, {})
|
24
|
-
#Zillabyte::Harness::Helper.check_emits("each", h._emits, @_streams) || @_branched
|
25
29
|
@_app._nodes << h
|
26
30
|
if(@_app._options[:command] == :info)
|
27
31
|
info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
|
@@ -73,9 +77,8 @@ class Zillabyte::Harness::Stream
|
|
73
77
|
|
74
78
|
def sink(&block)
|
75
79
|
h = Zillabyte::Harness::Sink.new()
|
76
|
-
#yield(h)
|
77
80
|
h.instance_eval(&block)
|
78
|
-
|
81
|
+
Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
|
79
82
|
@_app._nodes << h
|
80
83
|
if(@_app._options[:command] == :info)
|
81
84
|
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.0.
|
47
|
+
version: 0.0.18
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.0.
|
54
|
+
version: 0.0.18
|
55
55
|
description: The Official Zillabyte Gem
|
56
56
|
email:
|
57
57
|
- gem@zillabyte.com
|
@@ -74,7 +74,6 @@ files:
|
|
74
74
|
- ruby/lib/zillabyte/harness.rb
|
75
75
|
- ruby/lib/zillabyte/version.rb
|
76
76
|
- ruby/lib/zillabyte.rb
|
77
|
-
- ruby/README.md
|
78
77
|
homepage: http://www.zillabyte.com
|
79
78
|
licenses:
|
80
79
|
- MIT
|
@@ -95,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
94
|
version: '0'
|
96
95
|
requirements: []
|
97
96
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
97
|
+
rubygems_version: 2.0.7
|
99
98
|
signing_key:
|
100
99
|
specification_version: 4
|
101
100
|
summary: The Official Zillabyte Gem
|
data/ruby/README.md
DELETED