zillabyte 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +9 -9
- data/ruby/lib/zillabyte/harness/app.rb +11 -5
- data/ruby/lib/zillabyte/harness/each.rb +1 -5
- data/ruby/lib/zillabyte/harness/helper.rb +95 -214
- data/ruby/lib/zillabyte/harness/source.rb +1 -4
- data/ruby/lib/zillabyte/harness/stream.rb +8 -5
- data/ruby/lib/zillabyte/version.rb +1 -1
- metadata +5 -6
- data/ruby/README.md +0 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YWVjZWZiYTQwODcyODk2OGE3MzM1NWFiMzkwZmVkM2E3NzgyZWFkZg==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
7
|
-
|
6
|
+
MTczN2E4NzUzMjA2ZWFiMjFhZjJmODNiNGE4ZWRhNTViZjhmOGVmNQ==
|
7
|
+
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NWM3YmZkOWIyYTNiNzhhYmViMmRkMTkwY2E5NjhmMDA4NDEyMmVjYmNhNDk1
|
10
|
+
ZDQ4MmJmNTFjNjQxMzAyZjIzOGQ5YmM0NjgyMzhlNWMyMTMwZWFkNTA3NmVk
|
11
|
+
N2QyZjFhYWU5YTViZjYwN2UzYmVkMDU0YzEyYzAzMDNkYzI0ZGM=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
YzgxYjAwOTUzMTNiMWI5MGJkMjE0ZTViZGU2MTc2NDRlNDJhZGIyYjJiNTBj
|
14
|
+
NzhjOGYxMTE1ZTRjMjVmZGExZmE2MmM4OGJmOWNhYTFiZTkxZDg3NzQ5ZGVk
|
15
|
+
ODg4NmRhZmZjYjE5OGExZTUxYWI4ZWViZTE5NWM4ZWQ3Y2RhMDU=
|
@@ -1,12 +1,13 @@
|
|
1
1
|
require "json"
|
2
2
|
|
3
3
|
class Zillabyte::Harness::App
|
4
|
-
attr_accessor :_name, :_options, :_nodes, :_streams, :_info_file
|
4
|
+
attr_accessor :_name, :_options, :_nodes, :_streams, :_names, :_info_file
|
5
5
|
|
6
6
|
def self.build(name)
|
7
7
|
h = Zillabyte::Harness::App.new()
|
8
8
|
h._nodes = []
|
9
9
|
h._streams = {}
|
10
|
+
h._names = {}
|
10
11
|
h._name = name
|
11
12
|
Zillabyte::Harness::Helper.check_name("new", h._name, {})
|
12
13
|
h._options = Zillabyte::Harness::Helper.opt_parser()
|
@@ -23,23 +24,28 @@ class Zillabyte::Harness::App
|
|
23
24
|
# Are we given a block?
|
24
25
|
if block_given?
|
25
26
|
h.instance_eval(&block)
|
26
|
-
Zillabyte::Harness::Helper.check_name("source", h._name,
|
27
|
-
Zillabyte::Harness::Helper.
|
28
|
-
if
|
27
|
+
Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
|
28
|
+
Zillabyte::Harness::Helper.check_custom_source(h)
|
29
|
+
if h._emits
|
30
|
+
Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams)
|
31
|
+
else
|
29
32
|
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
30
33
|
end
|
31
34
|
# No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
|
32
35
|
else
|
33
36
|
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
37
|
+
Zillabyte::Harness::Helper.check_source_args(args)
|
34
38
|
h.matches(args[0])
|
35
39
|
end
|
36
40
|
@_nodes << h
|
37
41
|
if(@_options[:command] == :info)
|
38
|
-
info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits
|
42
|
+
info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
|
39
43
|
if(h._relation)
|
40
44
|
info_hash["relation"] = h._relation
|
41
45
|
elsif(h._matches)
|
42
46
|
info_hash["matches"] = h._matches
|
47
|
+
else
|
48
|
+
info_hash["end_cycle_policy"] = h._end_cycle_policy
|
43
49
|
end
|
44
50
|
Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
|
45
51
|
elsif(@_options[:command] == :execute and @_options[:name] == h._name)
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Zillabyte::Harness::Each
|
2
|
-
attr_accessor :
|
2
|
+
attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
|
3
3
|
|
4
4
|
def initialize()
|
5
5
|
@_name = "each_"+Zillabyte::Harness::Counter.get()
|
@@ -14,10 +14,6 @@ class Zillabyte::Harness::Each
|
|
14
14
|
@_emits = *v
|
15
15
|
end
|
16
16
|
|
17
|
-
def emit(*v)
|
18
|
-
@_has_emit = true
|
19
|
-
end
|
20
|
-
|
21
17
|
def prepare(&block)
|
22
18
|
@_prepare = block
|
23
19
|
end
|
@@ -35,6 +35,10 @@ class Zillabyte::Harness::Helper
|
|
35
35
|
exit(1)
|
36
36
|
end
|
37
37
|
|
38
|
+
def self.print_warning(msg)
|
39
|
+
STDERR.write(msg+"\n")
|
40
|
+
end
|
41
|
+
|
38
42
|
def self.check_name(operation, name, names)
|
39
43
|
ee = "Error in \"#{operation}\" at \"name\": \n\t "
|
40
44
|
|
@@ -51,92 +55,87 @@ class Zillabyte::Harness::Helper
|
|
51
55
|
|
52
56
|
end
|
53
57
|
|
54
|
-
def self.
|
55
|
-
ee = "Error in \"
|
58
|
+
def self.check_custom_source(source)
|
59
|
+
ee = "Error in \"source\": \n\t "
|
56
60
|
pp = @@_print_check_source
|
57
61
|
|
58
62
|
rm = !source._relation.nil?
|
59
63
|
mm = !source._matches.nil?
|
60
|
-
if( rm
|
64
|
+
if( rm or mm )
|
61
65
|
msg = "#{ee}"
|
62
|
-
msg += "A \"
|
66
|
+
msg += "A custom \"source\" may not contain a \"matches\" clause. To source from a relation, see the following. #{pp}"
|
63
67
|
Zillabyte::Harness::Helper.print_error(msg)
|
64
68
|
end
|
69
|
+
|
65
70
|
if(rm or mm)
|
66
|
-
if source.
|
67
|
-
msg = "#{ee}If \"
|
71
|
+
if source._begin_cycle or source._next_tuple
|
72
|
+
msg = "#{ee}If \"source\" has a \"matches\\relation\" clause, it may not have a \"begin_cycle\" or \"next_tuple\" block. #{pp}"
|
68
73
|
Zillabyte::Harness::Helper.print_error(msg)
|
69
74
|
end
|
70
75
|
end
|
76
|
+
|
77
|
+
allowed_policies = [:null_emit, :explicit]
|
78
|
+
if !(allowed_policies.member? source._end_cycle_policy)
|
79
|
+
msg = "#{ee}Invalid \"end_cycle_policy\": \"#{source._end_cycle_policy}\". The allowed policies are :null_emit and :explicit. #{pp}"
|
80
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
81
|
+
end
|
82
|
+
|
83
|
+
if !source._next_tuple
|
84
|
+
msg = "#{ee}A custom \"source\" must contain a \"next_tuple\" block. #{pp}"
|
85
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.check_source_args(args)
|
90
|
+
ee = "Error in \"source\": \n\t "
|
91
|
+
pp = @@_print_check_source
|
92
|
+
|
93
|
+
if(args.length != 1)
|
94
|
+
msg = "#{ee}Invalid number of arguments to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
|
95
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
96
|
+
end
|
97
|
+
|
98
|
+
if(!args[0].instance_of?(String) and !args[0].instance_of?(Array))
|
99
|
+
msg = "#{ee}Invalid argument to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
|
100
|
+
Zillabyte::Harness::Helper.print_error(msg)
|
101
|
+
end
|
71
102
|
end
|
72
103
|
|
73
104
|
def self.check_emits(operation, emits, streams)
|
74
105
|
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
75
|
-
pp =
|
76
|
-
nn = "stream"
|
106
|
+
pp = ""
|
77
107
|
|
78
108
|
if(!emits.instance_of?(Array))
|
79
|
-
msg = "#{ee}\"Emits\" must be a sequence of
|
109
|
+
msg = "#{ee}\"Emits\" must be a sequence of comma-separated STRINGS. #{pp}"
|
80
110
|
Zillabyte::Harness::Helper.print_error(msg)
|
81
111
|
end
|
82
112
|
n_emits = emits.length
|
83
113
|
if(n_emits == 0)
|
84
|
-
msg = "#{ee}Must emit at least one
|
114
|
+
msg = "#{ee}Must emit at least one stream, \"emits\" cannot be an empty. #{pp}"
|
85
115
|
Zillabyte::Harness::Helper.print_error(msg)
|
86
116
|
end
|
87
117
|
|
88
|
-
current_op_streams = {}
|
89
118
|
emits.each do |e|
|
90
119
|
if(!e.instance_of?(String) or (e =~ /^\w+$/).nil?)
|
91
|
-
msg = "#{ee}\"Emits\"
|
120
|
+
msg = "#{ee}\"Emits\" stream name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
|
92
121
|
Zillabyte::Harness::Helper.print_error(msg)
|
93
122
|
end
|
94
|
-
if(
|
95
|
-
msg = "#{ee}The
|
123
|
+
if(streams[e])
|
124
|
+
msg = "#{ee}The stream name \"#{e}\" is previously defined! #{pp}"
|
96
125
|
Zillabyte::Harness::Helper.print_error(msg)
|
97
126
|
end
|
98
|
-
|
99
|
-
end
|
100
|
-
if(n_emits > 1)
|
101
|
-
return true
|
102
|
-
else
|
103
|
-
return false
|
127
|
+
streams[e] = operation
|
104
128
|
end
|
105
|
-
|
106
129
|
end
|
107
130
|
|
108
|
-
def self.
|
109
|
-
ee = "Error in \"
|
110
|
-
pp =
|
111
|
-
if(operation == "aggregate")
|
112
|
-
pp = @@_print_check_group_by
|
113
|
-
end
|
131
|
+
def self.check_each(each)
|
132
|
+
ee = "Error in \"each\": \n\t "
|
133
|
+
pp = @@_print_check_each
|
114
134
|
|
115
|
-
|
116
|
-
|
117
|
-
msg = "#{ee}Group_by must be an ARRAY with at least one element! #{pp}"
|
135
|
+
if !each._execute
|
136
|
+
msg = "#{ee}An \"each\" must contain an \"execute\" block. #{pp}"
|
118
137
|
Zillabyte::Harness::Helper.print_error(msg)
|
119
|
-
end
|
120
|
-
|
121
|
-
if(operation == "aggregate")
|
122
|
-
# Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from source!
|
123
|
-
group_by.each do |f|
|
124
|
-
if(h._consumes)
|
125
|
-
fields = streams[h._consumes]
|
126
|
-
if(!fields.include?(f))
|
127
|
-
msg = "#{ee}Group_by field not in consumed stream \"#{h._consumes}\". #{pp}"
|
128
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
129
|
-
end
|
130
|
-
else
|
131
|
-
previous_emits = nodes.last._emits[0] #if no "consumes", then operation consumes the sole stream from the previous operation
|
132
|
-
if(!previous_emits[1].include?(f))
|
133
|
-
msg = "#{ee}Group_by field not in \"emits\" of previous relation \"#{previous_emits[0]}\". #{pp}"
|
134
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
135
|
-
end
|
136
|
-
end
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
138
|
+
end
|
140
139
|
end
|
141
140
|
|
142
141
|
def self.check_sink(sink, nodes)
|
@@ -174,18 +173,13 @@ class Zillabyte::Harness::Helper
|
|
174
173
|
columns.each do |col|
|
175
174
|
cname = col.keys()[0]
|
176
175
|
ctype = col[cname]
|
177
|
-
Zillabyte::Harness::Helper.check_sink_column_format(
|
176
|
+
Zillabyte::Harness::Helper.check_sink_column_format(cname,ctype,name)
|
178
177
|
end
|
179
178
|
end
|
180
179
|
|
181
|
-
def self.check_sink_column_format(
|
182
|
-
|
183
|
-
|
184
|
-
pp = @@_print_check_sink
|
185
|
-
elsif(operation.include?("simple"))
|
186
|
-
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
187
|
-
pp = @@_print_check_simple_app_emits
|
188
|
-
end
|
180
|
+
def self.check_sink_column_format(cname, ctype, relation_name)
|
181
|
+
ee = "Error in \"sink\" at \"column\": \n\t "
|
182
|
+
pp = @@_print_check_sink
|
189
183
|
|
190
184
|
if(!cname.instance_of?(String) or (cname =~ /^\w+$/).nil?)
|
191
185
|
msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in relation \"#{relation_name}\". #{pp}"
|
@@ -205,60 +199,6 @@ class Zillabyte::Harness::Helper
|
|
205
199
|
end
|
206
200
|
end
|
207
201
|
|
208
|
-
def self.check_sink_consumes(sink, streams)
|
209
|
-
ee = "Error in \"sink\" at \"consumes\": \n\t "
|
210
|
-
pp = @@_print_check_sink
|
211
|
-
|
212
|
-
name = sink._name
|
213
|
-
columns = sink._columns
|
214
|
-
consumes = sink._consumes
|
215
|
-
|
216
|
-
stream_fields = streams[consumes]
|
217
|
-
if(stream_fields.length != columns.length)
|
218
|
-
msg = "#{ee}Number of columns in \"sink\" differs from number of fields in the consumed stream at relation \"#{name}\". #{pp}"
|
219
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
220
|
-
end
|
221
|
-
columns.each do |col|
|
222
|
-
col_name = col.keys()[0]
|
223
|
-
if(!stream_fields.include?(col_name))
|
224
|
-
msg = "#{ee}The column \"#{col_name}\", is not emitted by the stream \"#{consumes}\". #{pp}"
|
225
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
226
|
-
end
|
227
|
-
end
|
228
|
-
end
|
229
|
-
|
230
|
-
def self.check_simple_app_emits(operation, emits)
|
231
|
-
ee = "Error in \"#{operation}\" at \"emits\": \n\t "
|
232
|
-
pp = @@_print_check_simple_app_emits
|
233
|
-
|
234
|
-
name = emits[0]
|
235
|
-
columns = emits[1]
|
236
|
-
if(!columns.instance_of?(Array))
|
237
|
-
msg = "#{ee}Field names must be an ARRAY of HASHES in relation \"#{name}\". #{pp}"
|
238
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
239
|
-
end
|
240
|
-
columns.each do |col|
|
241
|
-
if(!col.instance_of?(Hash))
|
242
|
-
msg = "#{ee}Fields names must be listed in HASH format in relation \"#{name}\". #{pp}"
|
243
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
244
|
-
end
|
245
|
-
colkeys = col.keys()
|
246
|
-
if(colkeys.length != 1)
|
247
|
-
msg = "#{ee}Each field must be a separate HASH with {field_name : data_type} in relation \"#{name}\". #{pp}"
|
248
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
249
|
-
end
|
250
|
-
colkey = colkeys[0]
|
251
|
-
colval = col[colkey]
|
252
|
-
Zillabyte::Harness::Helper.check_sink_column_format(operation,colkey,colval,name)
|
253
|
-
end
|
254
|
-
end
|
255
|
-
|
256
|
-
def self.print_simple_app_usage(msg)
|
257
|
-
ee = "Error in \"simple_app\": \n\t "
|
258
|
-
msg = "#{ee}#{msg} #{@@_print_check_simple_app} #{@@_print_check_simple_app_emits}"
|
259
|
-
Zillabyte::Harness::Helper.print_error(msg)
|
260
|
-
end
|
261
|
-
|
262
202
|
# Test helper...
|
263
203
|
def self.argv()
|
264
204
|
@_argv || ARGV
|
@@ -269,100 +209,61 @@ class Zillabyte::Harness::Helper
|
|
269
209
|
@_argv = v
|
270
210
|
end
|
271
211
|
|
272
|
-
@@_print_check_emits = <<-OUTPUT
|
273
|
-
"Emits" Syntax:
|
274
|
-
- "Emits" must be a non-empty ARRAY.
|
275
|
-
- Each element of "emits" must be an ARRAY of length = 2.
|
276
|
-
* The first element should be the unique stream name defined as a non-empty STRING with only alphanumeric and underscore characters.
|
277
|
-
* The second element should be an ARRAY of field names for that stream.
|
278
|
-
e.g.
|
279
|
-
emits = [ [ "stream_1", [ "field_11", "field_12", ... ] ],
|
280
|
-
[ "stream_2", [ "field_21", "field_22", ... ] ] ] .
|
281
|
-
- Stream and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
|
282
|
-
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
283
|
-
OUTPUT
|
284
|
-
|
285
|
-
|
286
|
-
@@_print_check_group_by =<<-OUTPUT
|
287
|
-
"Group_by" Syntax:
|
288
|
-
- "Group_by" must be a non-empty ARRAY.
|
289
|
-
- Each element of "group_by" must be a STRING corresponding to a field emitted by the previous operation or the stream
|
290
|
-
consumed by the aggregation operation.
|
291
|
-
OUTPUT
|
292
|
-
|
293
|
-
@@_print_check_simple_app_emits = <<-OUTPUT
|
294
|
-
"Emits" Syntax:
|
295
|
-
- "Emits" must be a non-empty ARRAY.
|
296
|
-
- Each element of "emits" must be an ARRAY of length = 2.
|
297
|
-
* The first element should be the unique relation name defined as a non-empty STRING with only alphanumeric and underscore characters.
|
298
|
-
* The second element should be an ARRAY of HASHES with field names and data types for that relation. e.g.
|
299
|
-
emits = [ [ "relation_1", [ {"field_11" => "type_11"}, {"field_12" => "type_12"}, ... ] ],
|
300
|
-
[ "relation_2", [ {"field_21" => "type_21"}, {"field_22" => "type_22"}, ... ] ] ] .
|
301
|
-
- Relation and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
|
302
|
-
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
303
|
-
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
304
|
-
OUTPUT
|
305
|
-
|
306
212
|
@@_print_check_sink = <<-OUTPUT
|
307
|
-
"Sink" Syntax:
|
308
|
-
|
309
|
-
Single stream:
|
310
|
-
app.sink do
|
213
|
+
\n\n"Sink" Syntax:
|
214
|
+
stream.sink do
|
311
215
|
name "name_of_relation"
|
312
|
-
column "field_1" :type_1
|
313
|
-
column "field_2" :type_2 ...
|
314
|
-
end
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
column "field_2" :type_2 ...
|
321
|
-
end
|
322
|
-
- "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
323
|
-
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
324
|
-
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
325
|
-
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
326
|
-
- If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
|
216
|
+
column "field_1", :type_1
|
217
|
+
column "field_2", :type_2 ...
|
218
|
+
end
|
219
|
+
- "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
|
220
|
+
- Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
|
221
|
+
- Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
|
222
|
+
- Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
|
223
|
+
- If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
|
327
224
|
* "Consumes" is the name of a stream emitted by an "each" or a "source" which the "sink" should save as a table.
|
328
225
|
* The columns specified in "sink" must match the fields emitted by the stream."
|
329
226
|
OUTPUT
|
330
227
|
|
331
228
|
@@_print_check_source = <<-OUTPUT
|
332
|
-
"Source" Syntax:
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
matches "sql_query or sxp"
|
338
|
-
end
|
339
|
-
Custom source:
|
229
|
+
\n\n"Source" Syntax:
|
230
|
+
Sourcing from a relation:
|
231
|
+
app.source("SQL query" or [SXP queries])
|
232
|
+
|
233
|
+
Custom source:
|
340
234
|
app.source do
|
341
|
-
name "name"
|
342
|
-
emits
|
343
|
-
|
344
|
-
|
345
|
-
|
235
|
+
name "name" \t\t\t\t\t => optional
|
236
|
+
emits "stream_1", "stream_2", ... \t\t => optional for single output stream
|
237
|
+
end_cycle_policy :null_emit OR :explicit \t => default :null_emit
|
238
|
+
begin_cycle |=block=| \t\t\t\t => optional if no initialization needed
|
239
|
+
next_tuple |=block=|
|
240
|
+
end
|
241
|
+
- The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
|
242
|
+
* :null_emit - end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
|
243
|
+
* :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
|
346
244
|
OUTPUT
|
347
245
|
|
348
246
|
@@_print_check_each = <<-OUTPUT
|
349
|
-
"Each" Syntax:
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
247
|
+
\n\n"Each" Syntax:
|
248
|
+
Simplified syntax:
|
249
|
+
stream.each do |tuple|
|
250
|
+
|=block=|
|
251
|
+
end
|
252
|
+
- This is equivalent to just specifying an \"execute\" block below.
|
253
|
+
- Note that this syntax only works for a single output stream.
|
254
|
+
|
255
|
+
Custom each:
|
256
|
+
stream.each do
|
257
|
+
name "name" \t\t\t\t\t => optional
|
258
|
+
emits "stream_1", "stream_2", ... \t\t => optional for single output stream
|
259
|
+
prepare |=block=| \t\t\t\t => optional if no initialization needed
|
260
|
+
execute |=block=|
|
357
261
|
end
|
358
|
-
- If there are multiple streams, "consumes" must be specified as a non-empty STRING!
|
359
|
-
* "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "each" operates on."
|
360
262
|
OUTPUT
|
361
263
|
|
362
264
|
@@_print_check_aggregate = <<-OUTPUT
|
363
|
-
"Aggregate" Syntax:
|
364
|
-
|
365
|
-
app.aggregate do
|
265
|
+
\n\n"Aggregate" Syntax:
|
266
|
+
stream.aggregate do
|
366
267
|
name "name" => optional
|
367
268
|
group_by ["field_1", "field_2", ...]
|
368
269
|
emits emits
|
@@ -371,29 +272,9 @@ OUTPUT
|
|
371
272
|
aggregate ...
|
372
273
|
end_group ...
|
373
274
|
end
|
374
|
-
|
275
|
+
- If there are multiple streams, "consumes" must be specified as a non-empty STRING!
|
375
276
|
* "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "aggregate" operates on.
|
376
277
|
OUTPUT
|
377
278
|
|
378
|
-
@@_print_check_simple_app = <<-OUTPUT
|
379
|
-
"Simple_app" Syntax:
|
380
|
-
- Simple_apps must be specified using the following syntax:
|
381
|
-
Sinking directly from a custom source:
|
382
|
-
Zillabyte.simple_app do
|
383
|
-
name "name"
|
384
|
-
emits emits
|
385
|
-
prepare ...
|
386
|
-
next_batch ...
|
387
|
-
end
|
388
|
-
Processing and sinking data from a relation:
|
389
|
-
Zillabyte.simple_app do
|
390
|
-
name "name"
|
391
|
-
matches "sql_query or sxp"
|
392
|
-
emits emits
|
393
|
-
prepare ...
|
394
|
-
execute ...
|
395
|
-
end
|
396
|
-
OUTPUT
|
397
|
-
|
398
279
|
@@_meta_names = ["id", "confidence", "since", "source"]
|
399
280
|
end
|
@@ -14,14 +14,18 @@ class Zillabyte::Harness::Stream
|
|
14
14
|
# Does the block take 0 arguments? If so it's not just an execute block.
|
15
15
|
if(block.arity == 0)
|
16
16
|
h.instance_eval(&block)
|
17
|
+
Zillabyte::Harness::Helper.check_name("each", h._name, @_app._names)
|
18
|
+
Zillabyte::Harness::Helper.check_each(h)
|
19
|
+
if h._emits
|
20
|
+
Zillabyte::Harness::Helper.check_emits("each", h._emits, @_app._streams)
|
21
|
+
else
|
22
|
+
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
23
|
+
end
|
17
24
|
# Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
|
18
25
|
else
|
19
26
|
h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
|
20
|
-
h._has_emit = true
|
21
27
|
h._execute = block
|
22
28
|
end
|
23
|
-
Zillabyte::Harness::Helper.check_name("each", h._name, {})
|
24
|
-
#Zillabyte::Harness::Helper.check_emits("each", h._emits, @_streams) || @_branched
|
25
29
|
@_app._nodes << h
|
26
30
|
if(@_app._options[:command] == :info)
|
27
31
|
info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
|
@@ -73,9 +77,8 @@ class Zillabyte::Harness::Stream
|
|
73
77
|
|
74
78
|
def sink(&block)
|
75
79
|
h = Zillabyte::Harness::Sink.new()
|
76
|
-
#yield(h)
|
77
80
|
h.instance_eval(&block)
|
78
|
-
|
81
|
+
Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
|
79
82
|
@_app._nodes << h
|
80
83
|
if(@_app._options[:command] == :info)
|
81
84
|
info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: 0.0.
|
47
|
+
version: 0.0.18
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: 0.0.
|
54
|
+
version: 0.0.18
|
55
55
|
description: The Official Zillabyte Gem
|
56
56
|
email:
|
57
57
|
- gem@zillabyte.com
|
@@ -74,7 +74,6 @@ files:
|
|
74
74
|
- ruby/lib/zillabyte/harness.rb
|
75
75
|
- ruby/lib/zillabyte/version.rb
|
76
76
|
- ruby/lib/zillabyte.rb
|
77
|
-
- ruby/README.md
|
78
77
|
homepage: http://www.zillabyte.com
|
79
78
|
licenses:
|
80
79
|
- MIT
|
@@ -95,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
95
94
|
version: '0'
|
96
95
|
requirements: []
|
97
96
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.
|
97
|
+
rubygems_version: 2.0.7
|
99
98
|
signing_key:
|
100
99
|
specification_version: 4
|
101
100
|
summary: The Official Zillabyte Gem
|
data/ruby/README.md
DELETED