zillabyte 0.0.17 → 0.0.18

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,15 +1,15 @@
1
1
  ---
2
2
  !binary "U0hBMQ==":
3
3
  metadata.gz: !binary |-
4
- YWU1ZGQ4YmZiNjUxNDdhZjExYWU5Y2IyMWJlNWE4MmU1MTM3YTYwNg==
4
+ YWVjZWZiYTQwODcyODk2OGE3MzM1NWFiMzkwZmVkM2E3NzgyZWFkZg==
5
5
  data.tar.gz: !binary |-
6
- NmVlNjAzNTIwMjRiODFjZmNmZTQzYzc0YTM0NWE4ZjFiN2QzOGUxZg==
7
- SHA512:
6
+ MTczN2E4NzUzMjA2ZWFiMjFhZjJmODNiNGE4ZWRhNTViZjhmOGVmNQ==
7
+ !binary "U0hBNTEy":
8
8
  metadata.gz: !binary |-
9
- NTNiMmYzMDBlNjJjMGVmYTQ3YjdkMTVjMmVmNDgyZTE1YmJiMTRhYmNmZjY3
10
- YTUzODQwZDZlZWViZGQ1NTkxNmE1NjAyNmQ4Nzg4YjljZjMzZjJmODY0MDJi
11
- OTdlYWYwOWRhOWRkZmU5OTY3ZmZjMTg3ZDZmMjEwZjMxMjAwNGY=
9
+ NWM3YmZkOWIyYTNiNzhhYmViMmRkMTkwY2E5NjhmMDA4NDEyMmVjYmNhNDk1
10
+ ZDQ4MmJmNTFjNjQxMzAyZjIzOGQ5YmM0NjgyMzhlNWMyMTMwZWFkNTA3NmVk
11
+ N2QyZjFhYWU5YTViZjYwN2UzYmVkMDU0YzEyYzAzMDNkYzI0ZGM=
12
12
  data.tar.gz: !binary |-
13
- OTRkMzYxYmM2YTY1MWYyNTllMTg5YjI1M2NhN2U5ZjBjMGE4MTM3MDExODdk
14
- ODQzMDc4MDkwY2NlZWE0Mzk1NDVjYjAxYzMyMGZiMDc0ZDVjYjI5NDE4ZDI5
15
- M2ExOWJjMWFkZTdkMmI2N2I4MWE4N2M5ZDM5ZTI2NzA3Nzc1NTk=
13
+ YzgxYjAwOTUzMTNiMWI5MGJkMjE0ZTViZGU2MTc2NDRlNDJhZGIyYjJiNTBj
14
+ NzhjOGYxMTE1ZTRjMjVmZGExZmE2MmM4OGJmOWNhYTFiZTkxZDg3NzQ5ZGVk
15
+ ODg4NmRhZmZjYjE5OGExZTUxYWI4ZWViZTE5NWM4ZWQ3Y2RhMDU=
@@ -1,12 +1,13 @@
1
1
  require "json"
2
2
 
3
3
  class Zillabyte::Harness::App
4
- attr_accessor :_name, :_options, :_nodes, :_streams, :_info_file
4
+ attr_accessor :_name, :_options, :_nodes, :_streams, :_names, :_info_file
5
5
 
6
6
  def self.build(name)
7
7
  h = Zillabyte::Harness::App.new()
8
8
  h._nodes = []
9
9
  h._streams = {}
10
+ h._names = {}
10
11
  h._name = name
11
12
  Zillabyte::Harness::Helper.check_name("new", h._name, {})
12
13
  h._options = Zillabyte::Harness::Helper.opt_parser()
@@ -23,23 +24,28 @@ class Zillabyte::Harness::App
23
24
  # Are we given a block?
24
25
  if block_given?
25
26
  h.instance_eval(&block)
26
- Zillabyte::Harness::Helper.check_name("source", h._name, {})
27
- Zillabyte::Harness::Helper.check_source("source", h)
28
- if(!h._emits)
27
+ Zillabyte::Harness::Helper.check_name("source", h._name, @_names)
28
+ Zillabyte::Harness::Helper.check_custom_source(h)
29
+ if h._emits
30
+ Zillabyte::Harness::Helper.check_emits("source", h._emits, @_streams)
31
+ else
29
32
  h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
30
33
  end
31
34
  # No block? Then we assume we're given sql or sxp and parse it using h.matches. Also give it a generated stream name.
32
35
  else
33
36
  h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
37
+ Zillabyte::Harness::Helper.check_source_args(args)
34
38
  h.matches(args[0])
35
39
  end
36
40
  @_nodes << h
37
41
  if(@_options[:command] == :info)
38
- info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits, "end_cycle_policy" => h._end_cycle_policy}
42
+ info_hash = {"name" => h._name, "type" => h._type, "emits" => h._emits}
39
43
  if(h._relation)
40
44
  info_hash["relation"] = h._relation
41
45
  elsif(h._matches)
42
46
  info_hash["matches"] = h._matches
47
+ else
48
+ info_hash["end_cycle_policy"] = h._end_cycle_policy
43
49
  end
44
50
  Zillabyte::Harness::Helper.write_hash_to_file(info_hash, @_info_file)
45
51
  elsif(@_options[:command] == :execute and @_options[:name] == h._name)
@@ -1,5 +1,5 @@
1
1
  class Zillabyte::Harness::Each
2
- attr_accessor :_has_emit, :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
2
+ attr_accessor :_name, :_type, :_emits, :_consumes, :_prepare, :_execute
3
3
 
4
4
  def initialize()
5
5
  @_name = "each_"+Zillabyte::Harness::Counter.get()
@@ -14,10 +14,6 @@ class Zillabyte::Harness::Each
14
14
  @_emits = *v
15
15
  end
16
16
 
17
- def emit(*v)
18
- @_has_emit = true
19
- end
20
-
21
17
  def prepare(&block)
22
18
  @_prepare = block
23
19
  end
@@ -35,6 +35,10 @@ class Zillabyte::Harness::Helper
35
35
  exit(1)
36
36
  end
37
37
 
38
+ def self.print_warning(msg)
39
+ STDERR.write(msg+"\n")
40
+ end
41
+
38
42
  def self.check_name(operation, name, names)
39
43
  ee = "Error in \"#{operation}\" at \"name\": \n\t "
40
44
 
@@ -51,92 +55,87 @@ class Zillabyte::Harness::Helper
51
55
 
52
56
  end
53
57
 
54
- def self.check_source(operation, source)
55
- ee = "Error in \"#{operation}\": \n\t "
58
+ def self.check_custom_source(source)
59
+ ee = "Error in \"source\": \n\t "
56
60
  pp = @@_print_check_source
57
61
 
58
62
  rm = !source._relation.nil?
59
63
  mm = !source._matches.nil?
60
- if( rm and mm )
64
+ if( rm or mm )
61
65
  msg = "#{ee}"
62
- msg += "A \"#{operation}\" may contain either a \"matches\" clause or a \"relation\" clause, not both. #{pp}"
66
+ msg += "A custom \"source\" may not contain a \"matches\" clause. To source from a relation, see the following. #{pp}"
63
67
  Zillabyte::Harness::Helper.print_error(msg)
64
68
  end
69
+
65
70
  if(rm or mm)
66
- if source._prepare or source._next_batch
67
- msg = "#{ee}If \"#{operation}\" has a \"matches\\relation\" clause, it may not have a \"prepare\" or \"next_batch\" block. #{pp}"
71
+ if source._begin_cycle or source._next_tuple
72
+ msg = "#{ee}If \"source\" has a \"matches\\relation\" clause, it may not have a \"begin_cycle\" or \"next_tuple\" block. #{pp}"
68
73
  Zillabyte::Harness::Helper.print_error(msg)
69
74
  end
70
75
  end
76
+
77
+ allowed_policies = [:null_emit, :explicit]
78
+ if !(allowed_policies.member? source._end_cycle_policy)
79
+ msg = "#{ee}Invalid \"end_cycle_policy\": \"#{source._end_cycle_policy}\". The allowed policies are :null_emit and :explicit. #{pp}"
80
+ Zillabyte::Harness::Helper.print_error(msg)
81
+ end
82
+
83
+ if !source._next_tuple
84
+ msg = "#{ee}A custom \"source\" must contain a \"next_tuple\" block. #{pp}"
85
+ Zillabyte::Harness::Helper.print_error(msg)
86
+ end
87
+ end
88
+
89
+ def self.check_source_args(args)
90
+ ee = "Error in \"source\": \n\t "
91
+ pp = @@_print_check_source
92
+
93
+ if(args.length != 1)
94
+ msg = "#{ee}Invalid number of arguments to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
95
+ Zillabyte::Harness::Helper.print_error(msg)
96
+ end
97
+
98
+ if(!args[0].instance_of?(String) and !args[0].instance_of?(Array))
99
+ msg = "#{ee}Invalid argument to \"source\". When sourcing from a relation, please supply a single SQL query-string or SXP query-array. #{pp}"
100
+ Zillabyte::Harness::Helper.print_error(msg)
101
+ end
71
102
  end
72
103
 
73
104
  def self.check_emits(operation, emits, streams)
74
105
  ee = "Error in \"#{operation}\" at \"emits\": \n\t "
75
- pp = @@_print_check_emits
76
- nn = "stream"
106
+ pp = ""
77
107
 
78
108
  if(!emits.instance_of?(Array))
79
- msg = "#{ee}\"Emits\" must be a sequence of strings at #{emits}. #{pp}"
109
+ msg = "#{ee}\"Emits\" must be a sequence of comma-separated STRINGS. #{pp}"
80
110
  Zillabyte::Harness::Helper.print_error(msg)
81
111
  end
82
112
  n_emits = emits.length
83
113
  if(n_emits == 0)
84
- msg = "#{ee}Must emit at least one #{nn}, \"emits\" cannot be an empty array. #{pp}"
114
+ msg = "#{ee}Must emit at least one stream, \"emits\" cannot be an empty. #{pp}"
85
115
  Zillabyte::Harness::Helper.print_error(msg)
86
116
  end
87
117
 
88
- current_op_streams = {}
89
118
  emits.each do |e|
90
119
  if(!e.instance_of?(String) or (e =~ /^\w+$/).nil?)
91
- msg = "#{ee}\"Emits\" #{nn} name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
120
+ msg = "#{ee}\"Emits\" stream name must be a non-empty STRING with only alphanumeric and underscore characters in \"#{e}\". #{pp}"
92
121
  Zillabyte::Harness::Helper.print_error(msg)
93
122
  end
94
- if(current_op_streams[e])
95
- msg = "#{ee}The #{nn} \"#{e[0]}\" is listed multiple times in the same \"emits\". #{pp}"
123
+ if(streams[e])
124
+ msg = "#{ee}The stream name \"#{e}\" is previously defined! #{pp}"
96
125
  Zillabyte::Harness::Helper.print_error(msg)
97
126
  end
98
- current_op_streams[e] = 1
99
- end
100
- if(n_emits > 1)
101
- return true
102
- else
103
- return false
127
+ streams[e] = operation
104
128
  end
105
-
106
129
  end
107
130
 
108
- def self.check_group_by(operation, h, nodes, streams)
109
- ee = "Error in \"#{operation}\" at \"group_by\": \n\t "
110
- pp = ""
111
- if(operation == "aggregate")
112
- pp = @@_print_check_group_by
113
- end
131
+ def self.check_each(each)
132
+ ee = "Error in \"each\": \n\t "
133
+ pp = @@_print_check_each
114
134
 
115
- group_by = h._group_by
116
- if(!group_by.instance_of?(Array) or group_by == [])
117
- msg = "#{ee}Group_by must be an ARRAY with at least one element! #{pp}"
135
+ if !each._execute
136
+ msg = "#{ee}An \"each\" must contain an \"execute\" block. #{pp}"
118
137
  Zillabyte::Harness::Helper.print_error(msg)
119
- end
120
-
121
- if(operation == "aggregate")
122
- # Can't check group_by fields for simple_aggregate because "matches" does not specify emitted fields from source!
123
- group_by.each do |f|
124
- if(h._consumes)
125
- fields = streams[h._consumes]
126
- if(!fields.include?(f))
127
- msg = "#{ee}Group_by field not in consumed stream \"#{h._consumes}\". #{pp}"
128
- Zillabyte::Harness::Helper.print_error(msg)
129
- end
130
- else
131
- previous_emits = nodes.last._emits[0] #if no "consumes", then operation consumes the sole stream from the previous operation
132
- if(!previous_emits[1].include?(f))
133
- msg = "#{ee}Group_by field not in \"emits\" of previous relation \"#{previous_emits[0]}\". #{pp}"
134
- Zillabyte::Harness::Helper.print_error(msg)
135
- end
136
- end
137
- end
138
- end
139
-
138
+ end
140
139
  end
141
140
 
142
141
  def self.check_sink(sink, nodes)
@@ -174,18 +173,13 @@ class Zillabyte::Harness::Helper
174
173
  columns.each do |col|
175
174
  cname = col.keys()[0]
176
175
  ctype = col[cname]
177
- Zillabyte::Harness::Helper.check_sink_column_format("sink",cname,ctype,name)
176
+ Zillabyte::Harness::Helper.check_sink_column_format(cname,ctype,name)
178
177
  end
179
178
  end
180
179
 
181
- def self.check_sink_column_format(operation, cname, ctype, relation_name)
182
- if(operation == "sink")
183
- ee = "Error in \"sink\" at \"column\": \n\t "
184
- pp = @@_print_check_sink
185
- elsif(operation.include?("simple"))
186
- ee = "Error in \"#{operation}\" at \"emits\": \n\t "
187
- pp = @@_print_check_simple_app_emits
188
- end
180
+ def self.check_sink_column_format(cname, ctype, relation_name)
181
+ ee = "Error in \"sink\" at \"column\": \n\t "
182
+ pp = @@_print_check_sink
189
183
 
190
184
  if(!cname.instance_of?(String) or (cname =~ /^\w+$/).nil?)
191
185
  msg = "#{ee}Field names must be non-empty STRINGS with only alphanumeric and underscore characters in relation \"#{relation_name}\". #{pp}"
@@ -205,60 +199,6 @@ class Zillabyte::Harness::Helper
205
199
  end
206
200
  end
207
201
 
208
- def self.check_sink_consumes(sink, streams)
209
- ee = "Error in \"sink\" at \"consumes\": \n\t "
210
- pp = @@_print_check_sink
211
-
212
- name = sink._name
213
- columns = sink._columns
214
- consumes = sink._consumes
215
-
216
- stream_fields = streams[consumes]
217
- if(stream_fields.length != columns.length)
218
- msg = "#{ee}Number of columns in \"sink\" differs from number of fields in the consumed stream at relation \"#{name}\". #{pp}"
219
- Zillabyte::Harness::Helper.print_error(msg)
220
- end
221
- columns.each do |col|
222
- col_name = col.keys()[0]
223
- if(!stream_fields.include?(col_name))
224
- msg = "#{ee}The column \"#{col_name}\", is not emitted by the stream \"#{consumes}\". #{pp}"
225
- Zillabyte::Harness::Helper.print_error(msg)
226
- end
227
- end
228
- end
229
-
230
- def self.check_simple_app_emits(operation, emits)
231
- ee = "Error in \"#{operation}\" at \"emits\": \n\t "
232
- pp = @@_print_check_simple_app_emits
233
-
234
- name = emits[0]
235
- columns = emits[1]
236
- if(!columns.instance_of?(Array))
237
- msg = "#{ee}Field names must be an ARRAY of HASHES in relation \"#{name}\". #{pp}"
238
- Zillabyte::Harness::Helper.print_error(msg)
239
- end
240
- columns.each do |col|
241
- if(!col.instance_of?(Hash))
242
- msg = "#{ee}Fields names must be listed in HASH format in relation \"#{name}\". #{pp}"
243
- Zillabyte::Harness::Helper.print_error(msg)
244
- end
245
- colkeys = col.keys()
246
- if(colkeys.length != 1)
247
- msg = "#{ee}Each field must be a separate HASH with {field_name : data_type} in relation \"#{name}\". #{pp}"
248
- Zillabyte::Harness::Helper.print_error(msg)
249
- end
250
- colkey = colkeys[0]
251
- colval = col[colkey]
252
- Zillabyte::Harness::Helper.check_sink_column_format(operation,colkey,colval,name)
253
- end
254
- end
255
-
256
- def self.print_simple_app_usage(msg)
257
- ee = "Error in \"simple_app\": \n\t "
258
- msg = "#{ee}#{msg} #{@@_print_check_simple_app} #{@@_print_check_simple_app_emits}"
259
- Zillabyte::Harness::Helper.print_error(msg)
260
- end
261
-
262
202
  # Test helper...
263
203
  def self.argv()
264
204
  @_argv || ARGV
@@ -269,100 +209,61 @@ class Zillabyte::Harness::Helper
269
209
  @_argv = v
270
210
  end
271
211
 
272
- @@_print_check_emits = <<-OUTPUT
273
- "Emits" Syntax:
274
- - "Emits" must be a non-empty ARRAY.
275
- - Each element of "emits" must be an ARRAY of length = 2.
276
- * The first element should be the unique stream name defined as a non-empty STRING with only alphanumeric and underscore characters.
277
- * The second element should be an ARRAY of field names for that stream.
278
- e.g.
279
- emits = [ [ "stream_1", [ "field_11", "field_12", ... ] ],
280
- [ "stream_2", [ "field_21", "field_22", ... ] ] ] .
281
- - Stream and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
282
- - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
283
- OUTPUT
284
-
285
-
286
- @@_print_check_group_by =<<-OUTPUT
287
- "Group_by" Syntax:
288
- - "Group_by" must be a non-empty ARRAY.
289
- - Each element of "group_by" must be a STRING corresponding to a field emitted by the previous operation or the stream
290
- consumed by the aggregation operation.
291
- OUTPUT
292
-
293
- @@_print_check_simple_app_emits = <<-OUTPUT
294
- "Emits" Syntax:
295
- - "Emits" must be a non-empty ARRAY.
296
- - Each element of "emits" must be an ARRAY of length = 2.
297
- * The first element should be the unique relation name defined as a non-empty STRING with only alphanumeric and underscore characters.
298
- * The second element should be an ARRAY of HASHES with field names and data types for that relation. e.g.
299
- emits = [ [ "relation_1", [ {"field_11" => "type_11"}, {"field_12" => "type_12"}, ... ] ],
300
- [ "relation_2", [ {"field_21" => "type_21"}, {"field_22" => "type_22"}, ... ] ] ] .
301
- - Relation and field names must all be non-empty STRINGS with only alphanumeric and underscore characters.
302
- - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
303
- - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
304
- OUTPUT
305
-
306
212
  @@_print_check_sink = <<-OUTPUT
307
- "Sink" Syntax:
308
- - Sinks must be specified using the following syntax:
309
- Single stream:
310
- app.sink do
213
+ \n\n"Sink" Syntax:
214
+ stream.sink do
311
215
  name "name_of_relation"
312
- column "field_1" :type_1
313
- column "field_2" :type_2 ...
314
- end
315
- Multiple streams:
316
- app.sink do
317
- name "relation_name"
318
- consumes "stream_consumed"
319
- column "field_1" :type_1
320
- column "field_2" :type_2 ...
321
- end
322
- - "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
323
- - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
324
- - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
325
- - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
326
- - If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
216
+ column "field_1", :type_1
217
+ column "field_2", :type_2 ...
218
+ end
219
+ - "Sink" relation "name" must be specified as a non-empty STRING with only alphanumeric and underscore characters!
220
+ - Field names must be non-empty STRINGS with only alphanumeric or underscore characters.
221
+ - Field names cannot be "v[number]", "id", "confidence", "since" or "source" which are reserved Zillabyte names.
222
+ - Field types must be SYMBOLS. The following types are allowed :string, :integer, :float, :double, and :boolean.
223
+ - If there are multiple streams, "consumes" must be specified for each sink as a non-empty STRING!
327
224
  * "Consumes" is the name of a stream emitted by an "each" or a "source" which the "sink" should save as a table.
328
225
  * The columns specified in "sink" must match the fields emitted by the stream."
329
226
  OUTPUT
330
227
 
331
228
  @@_print_check_source = <<-OUTPUT
332
- "Source" Syntax:
333
- - Sources must be specified using the following syntax:
334
- Sourcing from a relation:
335
- app.source do
336
- name "name" => optional
337
- matches "sql_query or sxp"
338
- end
339
- Custom source:
229
+ \n\n"Source" Syntax:
230
+ Sourcing from a relation:
231
+ app.source("SQL query" or [SXP queries])
232
+
233
+ Custom source:
340
234
  app.source do
341
- name "name" => optional
342
- emits emits
343
- prepare ...
344
- next_batch ...
345
- end
235
+ name "name" \t\t\t\t\t => optional
236
+ emits "stream_1", "stream_2", ... \t\t => optional for single output stream
237
+ end_cycle_policy :null_emit OR :explicit \t => default :null_emit
238
+ begin_cycle |=block=| \t\t\t\t => optional if no initialization needed
239
+ next_tuple |=block=|
240
+ end
241
+ - The "end_cycle_policy" is used to specify when a cycle should end. Two options are available:
242
+ * :null_emit - end the cycle when a field contains "nil" or when nothing is emitted from the "next_tuple" block.
243
+ * :explicit - the end of a cycle is explicitly declared in the "next_tuple" block. This is done by including the "end_cycle" keyword in the "next_tuple" block, e.g. end_cycle if @queue.nil?.
346
244
  OUTPUT
347
245
 
348
246
  @@_print_check_each = <<-OUTPUT
349
- "Each" Syntax:
350
- - Eaches must be specified using the following syntax:
351
- app.each do
352
- name "name" => optional
353
- emits emits
354
- consumes "consumed_stream" => optional if single stream
355
- prepare ...
356
- execute ...
247
+ \n\n"Each" Syntax:
248
+ Simplified syntax:
249
+ stream.each do |tuple|
250
+ |=block=|
251
+ end
252
+ - This is equivalent to just specifying an \"execute\" block below.
253
+ - Note that this syntax only works for a single output stream.
254
+
255
+ Custom each:
256
+ stream.each do
257
+ name "name" \t\t\t\t\t => optional
258
+ emits "stream_1", "stream_2", ... \t\t => optional for single output stream
259
+ prepare |=block=| \t\t\t\t => optional if no initialization needed
260
+ execute |=block=|
357
261
  end
358
- - If there are multiple streams, "consumes" must be specified as a non-empty STRING!
359
- * "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "each" operates on."
360
262
  OUTPUT
361
263
 
362
264
  @@_print_check_aggregate = <<-OUTPUT
363
- "Aggregate" Syntax:
364
- - Aggregates must be specified using the following syntax:
365
- app.aggregate do
265
+ \n\n"Aggregate" Syntax:
266
+ stream.aggregate do
366
267
  name "name" => optional
367
268
  group_by ["field_1", "field_2", ...]
368
269
  emits emits
@@ -371,29 +272,9 @@ OUTPUT
371
272
  aggregate ...
372
273
  end_group ...
373
274
  end
374
- - If there are multiple streams, "consumes" must be specified as a non-empty STRING!
275
+ - If there are multiple streams, "consumes" must be specified as a non-empty STRING!
375
276
  * "Consumes" is the name of a stream emitted by a preceding "each", "source" or "aggregate" which the current "aggregate" operates on.
376
277
  OUTPUT
377
278
 
378
- @@_print_check_simple_app = <<-OUTPUT
379
- "Simple_app" Syntax:
380
- - Simple_apps must be specified using the following syntax:
381
- Sinking directly from a custom source:
382
- Zillabyte.simple_app do
383
- name "name"
384
- emits emits
385
- prepare ...
386
- next_batch ...
387
- end
388
- Processing and sinking data from a relation:
389
- Zillabyte.simple_app do
390
- name "name"
391
- matches "sql_query or sxp"
392
- emits emits
393
- prepare ...
394
- execute ...
395
- end
396
- OUTPUT
397
-
398
279
  @@_meta_names = ["id", "confidence", "since", "source"]
399
280
  end
@@ -25,10 +25,7 @@ class Zillabyte::Harness::Source
25
25
  end
26
26
 
27
27
  def end_cycle_policy(v)
28
- allowed_policies = [:null_emit, :explicit]
29
- if(allowed_policies.member?(v))
30
- @_end_cycle_policy = v
31
- end
28
+ @_end_cycle_policy = v
32
29
  end
33
30
 
34
31
  def begin_cycle(&block)
@@ -14,14 +14,18 @@ class Zillabyte::Harness::Stream
14
14
  # Does the block take 0 arguments? If so it's not just an execute block.
15
15
  if(block.arity == 0)
16
16
  h.instance_eval(&block)
17
+ Zillabyte::Harness::Helper.check_name("each", h._name, @_app._names)
18
+ Zillabyte::Harness::Helper.check_each(h)
19
+ if h._emits
20
+ Zillabyte::Harness::Helper.check_emits("each", h._emits, @_app._streams)
21
+ else
22
+ h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
23
+ end
17
24
  # Takes more than 0? Then it takes |tuple| and is an execute block. Give it a generated stream name.
18
25
  else
19
26
  h._emits = ["stream_"+Zillabyte::Harness::Counter.get()]
20
- h._has_emit = true
21
27
  h._execute = block
22
28
  end
23
- Zillabyte::Harness::Helper.check_name("each", h._name, {})
24
- #Zillabyte::Harness::Helper.check_emits("each", h._emits, @_streams) || @_branched
25
29
  @_app._nodes << h
26
30
  if(@_app._options[:command] == :info)
27
31
  info_hash = {"name" => h._name, "type" => h._type, "consumes" => @_name}
@@ -73,9 +77,8 @@ class Zillabyte::Harness::Stream
73
77
 
74
78
  def sink(&block)
75
79
  h = Zillabyte::Harness::Sink.new()
76
- #yield(h)
77
80
  h.instance_eval(&block)
78
- # Zillabyte::Harness::Helper.check_sink(h, @_nodes)
81
+ Zillabyte::Harness::Helper.check_sink(h, @_app._nodes)
79
82
  @_app._nodes << h
80
83
  if(@_app._options[:command] == :info)
81
84
  info_hash = {"name" => h._name, "type" => h._type, "columns" => h._columns, "relation" => h._relation || h._name, "consumes" => @_name}
@@ -1,3 +1,3 @@
1
1
  module Zillabyte
2
- VERSION = "0.0.17" unless defined?(VERSION)
2
+ VERSION = "0.0.18" unless defined?(VERSION)
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: zillabyte
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.17
4
+ version: 0.0.18
5
5
  platform: ruby
6
6
  authors:
7
7
  - zillabyte
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-16 00:00:00.000000000 Z
11
+ date: 2014-04-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - ~>
46
46
  - !ruby/object:Gem::Version
47
- version: 0.0.17
47
+ version: 0.0.18
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - ~>
53
53
  - !ruby/object:Gem::Version
54
- version: 0.0.17
54
+ version: 0.0.18
55
55
  description: The Official Zillabyte Gem
56
56
  email:
57
57
  - gem@zillabyte.com
@@ -74,7 +74,6 @@ files:
74
74
  - ruby/lib/zillabyte/harness.rb
75
75
  - ruby/lib/zillabyte/version.rb
76
76
  - ruby/lib/zillabyte.rb
77
- - ruby/README.md
78
77
  homepage: http://www.zillabyte.com
79
78
  licenses:
80
79
  - MIT
@@ -95,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
95
94
  version: '0'
96
95
  requirements: []
97
96
  rubyforge_project:
98
- rubygems_version: 2.1.10
97
+ rubygems_version: 2.0.7
99
98
  signing_key:
100
99
  specification_version: 4
101
100
  summary: The Official Zillabyte Gem
data/ruby/README.md DELETED
@@ -1,2 +0,0 @@
1
- api.client.ruby
2
- ===============