zillabyte-cli 0.0.15 → 0.0.16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +9 -9
- data/lib/zillabyte-cli/version.rb +1 -1
- data/lib/zillabyte/api.rb +63 -6
- data/lib/zillabyte/api/logs.rb +21 -4
- data/lib/zillabyte/cli/flows.rb +45 -105
- data/lib/zillabyte/cli/help.rb +19 -11
- data/lib/zillabyte/cli/log_formatter.rb +103 -0
- data/lib/zillabyte/cli/query.rb +5 -5
- data/lib/zillabyte/cli/relations.rb +12 -12
- data/lib/zillabyte/cli/templates/ruby/simple_function.rb +12 -9
- data/zillabyte-cli.gemspec +3 -0
- metadata +32 -6
- data/lib/zillabyte/#api.rb# +0 -170
- data/lib/zillabyte/cli/#logs.rb# +0 -12
- data/lib/zillabyte/cli/templates/python/#simple_function.py# +0 -27
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MDRiNDYwNjlhZDQwYjc4NGNiNDZlNDkzYjE2NGNjZDhjYzMyNTdlMA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
7
|
-
|
6
|
+
YTUzMjI0MGQxYTkyZjE4ODQ0MDU4ODg3MTBkNWM0ZDliNGEwNWQ2NA==
|
7
|
+
!binary "U0hBNTEy":
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MmE4Y2RhZWY0ODE2ZTQ2MWIzMTgxMjk1Yzg4NzU0NzhhZWE2YzE2MWJhYzU5
|
10
|
+
ZTJmZjE0NGJmNTBlMjBhOTQ2YTVjNTZlNGQ1ODMzOGMwYTU4N2M4MjhiZTIy
|
11
|
+
MzlkYjc2ZmU2OTM2MjJlMTU1NTQxMDRiMzRmMTYwYTY2YTVhY2U=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
ZjAxMzlmOTEyYjI4ZTk2MmU2MWE1NThjMTQ5ODg4MmUxMjc1NDMzMmYxMjYz
|
14
|
+
ZmFlYWEyZjFlNTFmOGIwYTBhZDVkZWI5ZjZkNjI5NDBiMWExOGIzNDY3MTk0
|
15
|
+
ZGY5Yzc4Y2FjMTZhNDY0NTg2M2Q3ZTIwZjg1ZjA3MzA3Y2Y1YmQ=
|
data/lib/zillabyte/api.rb
CHANGED
@@ -62,13 +62,70 @@ class Zillabyte::API
|
|
62
62
|
|
63
63
|
def request(params, &block)
|
64
64
|
begin
|
65
|
-
|
66
|
-
#
|
67
|
-
|
68
|
-
|
69
|
-
|
65
|
+
|
66
|
+
# If a block is given, then we assume we're streaming (i.e. for logs)
|
67
|
+
if block_given?
|
68
|
+
|
69
|
+
# We tell excon to emit a chunk ever 50 bytes. This is an excon thing, doesn't
|
70
|
+
# affect the network IO chunk size
|
71
|
+
params[:chunk_size] = 50;
|
72
|
+
last_string = ""
|
73
|
+
|
74
|
+
# Executed on each chunk excon bubbles up
|
75
|
+
params[:response_block] = lambda do |chunk, remaining_bytes, total_bytes|
|
76
|
+
|
77
|
+
# assume single-line json, split by '\n'
|
78
|
+
last_string += chunk
|
79
|
+
ary = last_string.split("\n", -1)
|
80
|
+
|
81
|
+
# Are we at the end of a JSON record?
|
82
|
+
if ary.size > 1
|
83
|
+
last_string = ary.last + (if last_string.end_with?("\n") then "\n" else "" end)
|
84
|
+
json_records = ary[0..-2]
|
85
|
+
json_records.each do |json_record|
|
86
|
+
|
87
|
+
# Blank?
|
88
|
+
next if json_record.strip == ""
|
89
|
+
|
90
|
+
if json_record.start_with?("HTTP/1.1 50")
|
91
|
+
puts "internal server error (code 500)"
|
92
|
+
exit(1)
|
93
|
+
end
|
94
|
+
|
95
|
+
# Offer all the json records to the listening block
|
96
|
+
begin
|
97
|
+
block.call(JSON.parse(json_record))
|
98
|
+
rescue JSON::ParserError => e
|
99
|
+
error "unable to parse logs"
|
100
|
+
exit(1)
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
# Execute the request
|
109
|
+
response = @connection.request(params)
|
110
|
+
|
70
111
|
rescue Excon::Errors::HTTPStatusError => error
|
71
|
-
|
112
|
+
if @progress
|
113
|
+
|
114
|
+
# Show helpful error message
|
115
|
+
if error.response.body
|
116
|
+
begin
|
117
|
+
hash = JSON.parse(error.response.body)
|
118
|
+
if hash['error_message']
|
119
|
+
@progress.error hash['error_message']
|
120
|
+
end
|
121
|
+
rescue JSON::ParserError => e
|
122
|
+
# Trickle to generic message below
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
@progress.error "internal server error"
|
127
|
+
|
128
|
+
end
|
72
129
|
raise error
|
73
130
|
end
|
74
131
|
|
data/lib/zillabyte/api/logs.rb
CHANGED
@@ -5,17 +5,17 @@ class Zillabyte::API::Logs < Zillabyte::API::Base
|
|
5
5
|
|
6
6
|
|
7
7
|
# GET /flows/ID/logs
|
8
|
-
def get(flow_id, operation_id, options = {})
|
8
|
+
def get(flow_id, operation_id, options = {}, &block)
|
9
9
|
|
10
10
|
options = {
|
11
11
|
}.merge(options)
|
12
12
|
|
13
|
-
res = @api.request(
|
13
|
+
res = @api.request({
|
14
14
|
:expects => 200,
|
15
15
|
:method => :get,
|
16
|
-
:path => "/flows/#{flow_id}/logs/#{operation_id}",
|
16
|
+
:path => "/flows/#{flow_id}/logs/#{operation_id}/stream",
|
17
17
|
:body => options.to_json
|
18
|
-
)
|
18
|
+
}, &block)
|
19
19
|
|
20
20
|
res.body
|
21
21
|
|
@@ -23,6 +23,23 @@ class Zillabyte::API::Logs < Zillabyte::API::Base
|
|
23
23
|
|
24
24
|
|
25
25
|
|
26
|
+
def get_startup(flow_id, operation_id, options = {}, &block)
|
27
|
+
|
28
|
+
options = {
|
29
|
+
}.merge(options)
|
30
|
+
|
31
|
+
res = @api.request({
|
32
|
+
:expects => 200,
|
33
|
+
:method => :get,
|
34
|
+
:path => "/flows/#{flow_id}/logs/#{operation_id}/startup",
|
35
|
+
:body => options.to_json
|
36
|
+
}, &block)
|
37
|
+
|
38
|
+
res.body
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
|
26
43
|
|
27
44
|
|
28
45
|
end
|
data/lib/zillabyte/cli/flows.rb
CHANGED
@@ -63,23 +63,22 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
63
63
|
end
|
64
64
|
|
65
65
|
display "Starting up your flow...please wait..."
|
66
|
-
|
67
|
-
while true
|
68
|
-
all_logs = api.logs.get(res['id'], "_ALL_", {:push => true})
|
69
|
-
all_logs.each_pair do |op, logs|
|
70
|
-
next if logs.length == 0
|
71
|
-
difference = previous_logs[op] ? logs - previous_logs[op] : logs
|
72
|
-
difference.each do |log|
|
73
|
-
if ["start_up", "error"].include?(log["category"])
|
74
|
-
display log["line"]
|
75
|
-
end
|
76
|
-
exit(0) if log["line"].downcase.include?("flow deployed")
|
77
|
-
end
|
78
|
-
previous_logs[op] = logs
|
79
|
-
end
|
80
|
-
sleep(0.5)
|
81
|
-
end
|
66
|
+
sleep(2) # wait for kill command
|
82
67
|
|
68
|
+
lf = LogFormatter::Startup.new
|
69
|
+
api.logs.get_startup(res['id'], "_ALL_", {:push => true}) do |hash|
|
70
|
+
|
71
|
+
# Error?
|
72
|
+
error hash['error_message'] if hash['error']
|
73
|
+
|
74
|
+
# Print it
|
75
|
+
lf.print_log_line(hash)
|
76
|
+
|
77
|
+
# Exit when we get the 'done' message
|
78
|
+
# exit(0) if (hash['line'] || '').downcase.include?("flow deployed")
|
79
|
+
|
80
|
+
end
|
81
|
+
|
83
82
|
end
|
84
83
|
alias_command "push", "flows:push"
|
85
84
|
|
@@ -87,7 +86,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
87
86
|
|
88
87
|
# flows:pull ID DIR
|
89
88
|
#
|
90
|
-
# pulls a flow source to a directory.
|
89
|
+
# pulls a flow source to a directory.
|
91
90
|
#
|
92
91
|
# --force # pulls even if the directory exists
|
93
92
|
#
|
@@ -227,14 +226,12 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
227
226
|
#
|
228
227
|
# streams logs from the distributed workers
|
229
228
|
#
|
230
|
-
# -t, --tail # continuously watches for new results
|
231
229
|
# -v, --verbose LEVEL # sets the verbosity (error, info, debug) (default: info)
|
232
230
|
#
|
233
231
|
def logs
|
234
232
|
|
235
233
|
flow_id = options[:flow] || shift_argument
|
236
234
|
operation_id = options[:operation] || shift_argument || '_ALL_'
|
237
|
-
tail = options[:tail]
|
238
235
|
category = options[:verbose] || '_ALL_'
|
239
236
|
carry_settings = {
|
240
237
|
:category => category
|
@@ -247,17 +244,13 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
247
244
|
end
|
248
245
|
|
249
246
|
display "Retrieving logs for flow ##{flow_id}...please wait..."
|
250
|
-
|
247
|
+
lf = LogFormatter::Operation.new
|
248
|
+
self.api.logs.get(flow_id, operation_id, api_options) do |line|
|
251
249
|
|
252
|
-
|
253
|
-
|
254
|
-
carry_settings = show_log_output(res, carry_settings)
|
255
|
-
api_options[:since] = carry_settings[:since]
|
250
|
+
error line['error_message'] if line['error']
|
251
|
+
lf.print_log_line(line)
|
256
252
|
|
257
|
-
|
258
|
-
sleep(5) # HACK
|
259
|
-
end
|
260
|
-
end while(tail)
|
253
|
+
end
|
261
254
|
|
262
255
|
end
|
263
256
|
alias_command "logs", "flows:logs"
|
@@ -307,7 +300,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
307
300
|
end
|
308
301
|
|
309
302
|
|
310
|
-
# flows:test
|
303
|
+
# flows:test
|
311
304
|
#
|
312
305
|
# tests a local flow with sample data
|
313
306
|
#
|
@@ -349,6 +342,14 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
349
342
|
write_stream.flush
|
350
343
|
end
|
351
344
|
|
345
|
+
def truncate_message(msg)
|
346
|
+
return msg if(!msg.instance_of?(String))
|
347
|
+
t_length = 50 # truncates entries to this length
|
348
|
+
m_length = msg.length
|
349
|
+
msg_out = m_length > t_length ? msg[0..t_length-3]+"..." : msg
|
350
|
+
msg_out
|
351
|
+
end
|
352
|
+
|
352
353
|
def handshake(write_stream, read_stream, node, color)
|
353
354
|
begin
|
354
355
|
write_message write_stream, "{\"pidDir\": \"/tmp\"}\n"
|
@@ -361,7 +362,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
361
362
|
|
362
363
|
|
363
364
|
# INIT
|
364
|
-
test_data = options[:test_data] || shift_argument
|
365
365
|
dir = options[:dir] || Dir.pwd
|
366
366
|
|
367
367
|
meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, self, {:test => true})
|
@@ -397,10 +397,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
397
397
|
display "#{name} - #{msg}".colorize(override_color || color)
|
398
398
|
end
|
399
399
|
|
400
|
-
# A
|
401
|
-
if type == "
|
400
|
+
# A Source?
|
401
|
+
if type == "source"
|
402
402
|
|
403
|
-
# A
|
403
|
+
# A source from relation?
|
404
404
|
if node['matches'] or node["relation"]
|
405
405
|
matches = node['matches'] || (node["relation"]["query"])
|
406
406
|
op_display.call "Grabbing remote data"
|
@@ -426,6 +426,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
426
426
|
end
|
427
427
|
end
|
428
428
|
read_msg = {"tuple" => values, "meta" => meta, "column_aliases" => column_aliases}.to_json
|
429
|
+
values = Hash[values.map{|k, v| [truncate_message(k), truncate_message(v)]}]
|
429
430
|
op_display.call "emit tuple: #{values} #{meta}"
|
430
431
|
stream_messages[default_stream] ||= []
|
431
432
|
stream_messages[default_stream] << read_msg
|
@@ -436,7 +437,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
436
437
|
|
437
438
|
else
|
438
439
|
|
439
|
-
# A regular
|
440
|
+
# A regular source..
|
440
441
|
stream_messages[default_stream] ||= []
|
441
442
|
batches.times do |i|
|
442
443
|
stream_messages[default_stream] << "{\"command\": \"next\"}\n"
|
@@ -580,9 +581,10 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
580
581
|
# Make it human-understable
|
581
582
|
write_json = JSON.parse(write_msg)
|
582
583
|
if write_json['tuple']
|
583
|
-
|
584
|
+
display_hash = Hash[write_json['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
|
585
|
+
op_display.call "receiving: #{display_hash}"
|
584
586
|
elsif write_json['command'] == 'next'
|
585
|
-
op_display.call "starting next
|
587
|
+
op_display.call "starting next source batch"
|
586
588
|
else
|
587
589
|
puts write_json
|
588
590
|
end
|
@@ -628,7 +630,8 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
628
630
|
emit_stream = obj['stream'] || default_stream
|
629
631
|
stream_messages[emit_stream] ||= []
|
630
632
|
stream_messages[emit_stream] << next_msg.to_json
|
631
|
-
|
633
|
+
display_hash = Hash[obj['tuple'].map{|k, v| [truncate_message(k), truncate_message(v)]}]
|
634
|
+
op_display.call "emitted: #{display_hash} to #{emit_stream}"
|
632
635
|
elsif obj['command'] == 'log'
|
633
636
|
op_display.call "log: #{obj['msg']}"
|
634
637
|
else
|
@@ -671,7 +674,9 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
671
674
|
# We need to check here to see if we need to turn on multistream mode.
|
672
675
|
# Note that we still need the redundant checks above, in case the user
|
673
676
|
# didn't honor the emits contract.
|
674
|
-
|
677
|
+
if node["emits"] && node["emits"].size() > 1
|
678
|
+
split_branches = true
|
679
|
+
end
|
675
680
|
end
|
676
681
|
end
|
677
682
|
|
@@ -709,7 +714,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
709
714
|
# runs a local flow with live data
|
710
715
|
#
|
711
716
|
# --config CONFIG_FILE # use the given config file
|
712
|
-
#
|
717
|
+
# HIDDEN:
|
713
718
|
def live_run
|
714
719
|
|
715
720
|
name = options[:name] || shift_argument
|
@@ -735,7 +740,7 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
735
740
|
# outputs the info for the flow in the dir.
|
736
741
|
#
|
737
742
|
# --pretty # Pretty prints the info output
|
738
|
-
#
|
743
|
+
#
|
739
744
|
def info
|
740
745
|
info_file = SecureRandom.uuid
|
741
746
|
cmd = command("--info --file #{info_file}")
|
@@ -785,71 +790,6 @@ class Zillabyte::Command::Flows < Zillabyte::Command::Base
|
|
785
790
|
|
786
791
|
private
|
787
792
|
|
788
|
-
def show_log_output(lines, settings = {})
|
789
|
-
|
790
|
-
max_category_size = 0
|
791
|
-
settings["start_up"] = :green
|
792
|
-
settings["kill"] = :yellow
|
793
|
-
settings["ipc"] = :cyan
|
794
|
-
settings["error"] = :red
|
795
|
-
settings["system"] = :white
|
796
|
-
settings["run"] = :magenta
|
797
|
-
all_lines = []
|
798
|
-
|
799
|
-
lines.each_pair do |operation, lines|
|
800
|
-
lines.each do |line_h|
|
801
|
-
|
802
|
-
line = line_h['line']
|
803
|
-
category = line_h['category']
|
804
|
-
date = line_h['date']
|
805
|
-
|
806
|
-
all_lines << {
|
807
|
-
:line => line,
|
808
|
-
:category => category,
|
809
|
-
:color => settings[category],
|
810
|
-
:date => date
|
811
|
-
}
|
812
|
-
max_category_size = [max_category_size, category.size].max
|
813
|
-
|
814
|
-
if settings[:since]
|
815
|
-
settings[:since] = [date, settings[:since]].max
|
816
|
-
else
|
817
|
-
settings[:since] = date
|
818
|
-
end
|
819
|
-
|
820
|
-
end
|
821
|
-
|
822
|
-
end
|
823
|
-
|
824
|
-
all_lines.sort! do |a,b|
|
825
|
-
a[:date] <=> b[:date]
|
826
|
-
end
|
827
|
-
|
828
|
-
requested_category = settings[:category]
|
829
|
-
|
830
|
-
all_lines.each do |h|
|
831
|
-
color = h[:color]
|
832
|
-
|
833
|
-
case h[:category]
|
834
|
-
when nil
|
835
|
-
next
|
836
|
-
when 'start_up'
|
837
|
-
next if ['kill', 'run', 'ipc'].member?(requested_category)
|
838
|
-
when 'run'
|
839
|
-
next if ['kill', 'start_up', 'ipc'].member?(requested_category)
|
840
|
-
when 'ipc'
|
841
|
-
next if ['kill', 'start_up', 'run'].member?(requested_category)
|
842
|
-
when 'kill'
|
843
|
-
next if ['start_up', 'run', 'ipc'].member?(requested_category)
|
844
|
-
end
|
845
|
-
|
846
|
-
display "#{h[:category].rjust(max_category_size)} - #{h[:date]} UTC - #{h[:line]}".colorize(color)
|
847
|
-
end
|
848
|
-
|
849
|
-
settings
|
850
|
-
end
|
851
|
-
|
852
|
-
|
853
793
|
|
854
794
|
|
855
795
|
def command(arg="--execute_live", ignore_stderr = false)
|
data/lib/zillabyte/cli/help.rb
CHANGED
@@ -8,8 +8,16 @@ class Zillabyte::Command::Help < Zillabyte::Command::Base
|
|
8
8
|
|
9
9
|
|
10
10
|
def index(*direct_args)
|
11
|
+
|
11
12
|
if command = args.shift || direct_args.shift
|
12
|
-
|
13
|
+
if command == "aliases"
|
14
|
+
puts
|
15
|
+
puts "Useful aliases: "
|
16
|
+
puts
|
17
|
+
summary_for_aliases(Zillabyte::Command.command_aliases)
|
18
|
+
else
|
19
|
+
help_for_command(command)
|
20
|
+
end
|
13
21
|
else
|
14
22
|
help_for_root
|
15
23
|
end
|
@@ -18,6 +26,7 @@ class Zillabyte::Command::Help < Zillabyte::Command::Base
|
|
18
26
|
alias_command "-h", "help"
|
19
27
|
alias_command "--help", "help"
|
20
28
|
|
29
|
+
|
21
30
|
private
|
22
31
|
|
23
32
|
def commands_for_namespace(name)
|
@@ -74,13 +83,14 @@ private
|
|
74
83
|
end
|
75
84
|
|
76
85
|
def summary_for_aliases(command_aliases)
|
77
|
-
|
78
|
-
|
79
|
-
|
86
|
+
used_aliases = command_aliases.select{|c,a| !c.include?("-") and !c.include?(":") and !skip_command?(commands[a])}
|
87
|
+
a_size = longest(used_aliases.map { |al,al_to| al }) + 2
|
88
|
+
a_to_size = longest(used_aliases.map { |al,al_to| al_to }) + 2
|
89
|
+
used_aliases.sort_by{|k,v| v}.each do |command_alias, alias_to|
|
80
90
|
if command_alias.include?("-") or command_alias.include?(":")
|
81
91
|
next #skip -h, -help and live-run etc.
|
82
92
|
end
|
83
|
-
puts command_alias.ljust(a_size) + " --> " + alias_to.ljust(a_to_size
|
93
|
+
puts command_alias.ljust(a_size) + " --> " + alias_to.ljust(a_to_size) + " # " + commands[alias_to][:summary]
|
84
94
|
end
|
85
95
|
end
|
86
96
|
|
@@ -95,11 +105,9 @@ private
|
|
95
105
|
puts
|
96
106
|
summary_for_namespaces(additional_namespaces)
|
97
107
|
puts
|
98
|
-
puts "
|
99
|
-
puts
|
100
|
-
summary_for_aliases(Zillabyte::Command.command_aliases)
|
108
|
+
puts "Type \"zillabyte help aliases\" for shortcuts to common commands."
|
101
109
|
puts
|
102
|
-
puts "
|
110
|
+
puts "Visit our documentation for more help: http://docs.zillabyte.com. "
|
103
111
|
puts
|
104
112
|
end
|
105
113
|
|
@@ -107,7 +115,7 @@ private
|
|
107
115
|
namespace_commands = commands_for_namespace(name)
|
108
116
|
|
109
117
|
unless namespace_commands.empty?
|
110
|
-
size = longest(namespace_commands.map { |c| c[:banner] })
|
118
|
+
size = longest(namespace_commands.select{|c| !skip_command?(c)}.map { |c| c[:banner] })
|
111
119
|
namespace_commands.sort_by { |c| c[:banner].to_s }.each do |command|
|
112
120
|
next if skip_command?(command)
|
113
121
|
command[:summary] ||= legacy_help_for_command(command[:command])
|
@@ -141,10 +149,10 @@ private
|
|
141
149
|
end
|
142
150
|
|
143
151
|
if !namespace_commands.empty?
|
144
|
-
puts "Additional commands, type \"zillabyte help COMMAND\" for more details:"
|
145
152
|
puts
|
146
153
|
help_for_namespace(name)
|
147
154
|
puts
|
155
|
+
puts "To see additional help for each of the commands, type \"zillabyte help COMMAND\"."
|
148
156
|
elsif command.nil?
|
149
157
|
error "#{name} is not a zillabyte command. See `zillabyte help`."
|
150
158
|
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
class LogFormatter
|
2
|
+
|
3
|
+
########################################################
|
4
|
+
## Base / Generic ######################################
|
5
|
+
########################################################
|
6
|
+
|
7
|
+
class Base
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@all_colors ||= [:green, :yellow, :magenta, :cyan, :light_black, :light_green, :light_yellow, :light_blue, :light_magenta, :light_cyan]
|
11
|
+
@color_map = {
|
12
|
+
'flow' => :white
|
13
|
+
}
|
14
|
+
end
|
15
|
+
|
16
|
+
def print_log_line(hash)
|
17
|
+
|
18
|
+
# Dno't show heartbeats
|
19
|
+
return if hash['heartbeat']
|
20
|
+
|
21
|
+
# Init
|
22
|
+
line = hash['line']
|
23
|
+
category = hash['category']
|
24
|
+
date = hash['date']
|
25
|
+
name = hash['name']
|
26
|
+
index = hash['index']
|
27
|
+
|
28
|
+
# Don't display 'hidden' lines
|
29
|
+
if category == 'hidden'
|
30
|
+
|
31
|
+
handle_hidden(name, index, category, date, line);
|
32
|
+
|
33
|
+
else
|
34
|
+
|
35
|
+
# Print it out
|
36
|
+
display(name, index, category, date, line)
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
protected
|
45
|
+
|
46
|
+
def handle_hidden(operation_name, operation_index, category, date, line)
|
47
|
+
end
|
48
|
+
|
49
|
+
def display(operation_name, operation_index, category, date, line)
|
50
|
+
puts "#{get_name(operation_name, operation_index).rjust(9).colorize(color_for(operation_name))} - #{line}"
|
51
|
+
end
|
52
|
+
|
53
|
+
|
54
|
+
def get_name(operation_name, operation_index)
|
55
|
+
if operation_index and operation_index != ""
|
56
|
+
return "#{operation_name}.#{operation_index}"
|
57
|
+
else
|
58
|
+
return operation_name
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def color_for(operation_name)
|
63
|
+
@color_map[operation_name] ||= @all_colors[ @color_map.size % @all_colors.size ]
|
64
|
+
@color_map[operation_name]
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
########################################################
|
71
|
+
## Startup #############################################
|
72
|
+
########################################################
|
73
|
+
|
74
|
+
class Startup < Base
|
75
|
+
|
76
|
+
def display(operation_name, operation_index, category, date, line)
|
77
|
+
puts "#{get_name(operation_name, operation_index).rjust(15).colorize(color_for(operation_name))} : #{line}"
|
78
|
+
end
|
79
|
+
|
80
|
+
def handle_hidden(operation_name, operation_index, category, date, line)
|
81
|
+
case line
|
82
|
+
when 'REGISTER_FLOW_DEPLOYED', 'REGISTER_FLOW_TIMEOUT'
|
83
|
+
# done reading the logs
|
84
|
+
exit(0)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
########################################################
|
92
|
+
## Operation Colorized #################################
|
93
|
+
########################################################
|
94
|
+
|
95
|
+
class Operation < Base
|
96
|
+
# TODO: color'ify each operation
|
97
|
+
end
|
98
|
+
|
99
|
+
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
|
data/lib/zillabyte/cli/query.rb
CHANGED
@@ -163,9 +163,9 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
163
163
|
alias_command "sql", "query:sql"
|
164
164
|
|
165
165
|
|
166
|
-
# query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET
|
166
|
+
# query:pull:s3 QUERY S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
|
167
167
|
#
|
168
|
-
# pulls query data to S3_BUCKET/
|
168
|
+
# pulls query data to S3_BUCKET/FILE_KEY/part***.gz
|
169
169
|
#
|
170
170
|
def pull_to_s3
|
171
171
|
|
@@ -175,14 +175,14 @@ class Zillabyte::Command::Query < Zillabyte::Command::Base
|
|
175
175
|
user_s3_access_key = options[:s3_access_key] || shift_argument
|
176
176
|
user_s3_secret = options[:s3_secret] || shift_argument
|
177
177
|
user_s3_bucket = options[:s3_bucket] || shift_argument
|
178
|
-
|
178
|
+
user_s3_file_key = options[:s3_file_key] || shift_argument
|
179
179
|
error "no s3 access key provided" if user_s3_access_key.nil?
|
180
180
|
error "no s3 secret provided" if user_s3_secret.nil?
|
181
181
|
error "no s3 bucket provided" if user_s3_bucket.nil?
|
182
|
-
error "no s3 file path specified" if
|
182
|
+
error "no s3 file path specified" if user_s3_file_key.nil?
|
183
183
|
|
184
184
|
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
185
|
-
:s3_bucket => user_s3_bucket, :
|
185
|
+
:s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
|
186
186
|
|
187
187
|
res = self.api.queries.pull_to_s3(query, s3_params)
|
188
188
|
display "downloading query results to s3://#{res["s3_bucket"]}/#{res["s3_file_key"]}/"
|
@@ -82,7 +82,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
82
82
|
# --schema SCHEMA # Column names and types in the format "field_1:type_1,field_2:type_2,..."
|
83
83
|
# --public SCOPE # Make the relation public
|
84
84
|
# --file FILE # A data file
|
85
|
-
# --type
|
85
|
+
# --type TYPE # File format type, defaults to csv
|
86
86
|
# --description DESCRIPTION # Description of relation contents
|
87
87
|
# --aliases ALIASES # Relation name aliases in the format "alias_1,alias_2,..."
|
88
88
|
#
|
@@ -122,7 +122,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
122
122
|
#
|
123
123
|
# adds data to an existing relation
|
124
124
|
#
|
125
|
-
# --type TYPE #
|
125
|
+
# --type TYPE # File format type, defaults to csv
|
126
126
|
#
|
127
127
|
def append
|
128
128
|
|
@@ -149,7 +149,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
149
149
|
#
|
150
150
|
# pulls relation data into OUTPUT.gz
|
151
151
|
#
|
152
|
-
# --cycle_id [cycle_id] #
|
152
|
+
# --cycle_id [cycle_id] # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
|
153
153
|
#
|
154
154
|
def pull
|
155
155
|
|
@@ -157,7 +157,7 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
157
157
|
file = options[:file] || shift_argument
|
158
158
|
error "no id given" if id.nil?
|
159
159
|
error "no output file given" if file.nil?
|
160
|
-
file = file
|
160
|
+
file = "#{file}.gz" unless File.extname(file) == ".gz"
|
161
161
|
|
162
162
|
res = self.api.data.pull(id, options)
|
163
163
|
if(res["uri"])
|
@@ -194,11 +194,11 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
194
194
|
end
|
195
195
|
alias_command "rl:pull", "relations:pull"
|
196
196
|
|
197
|
-
# relations:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET
|
197
|
+
# relations:pull:s3 ID S3_KEY S3_SECRET S3_BUCKET s3_FILE_KEY
|
198
198
|
#
|
199
|
-
# pulls relation data to S3_BUCKET/
|
199
|
+
# pulls relation data to S3_BUCKET/FILE_KEY/part***.gz
|
200
200
|
#
|
201
|
-
# --cycle_id [cycle_id] #
|
201
|
+
# --cycle_id [cycle_id] # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
|
202
202
|
#
|
203
203
|
def pull_to_s3
|
204
204
|
|
@@ -208,14 +208,14 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
208
208
|
user_s3_access_key = options[:s3_access_key] || shift_argument
|
209
209
|
user_s3_secret = options[:s3_secret] || shift_argument
|
210
210
|
user_s3_bucket = options[:s3_bucket] || shift_argument
|
211
|
-
|
211
|
+
user_s3_file_key = options[:s3_file_key] || shift_argument
|
212
212
|
error "no s3 access key provided" if user_s3_access_key.nil?
|
213
213
|
error "no s3 secret provided" if user_s3_secret.nil?
|
214
214
|
error "no s3 bucket provided" if user_s3_bucket.nil?
|
215
|
-
error "no s3 file path specified" if
|
215
|
+
error "no s3 file path specified" if user_s3_file_key.nil?
|
216
216
|
|
217
217
|
s3_params = {:s3_access_key => user_s3_access_key, :s3_secret => user_s3_secret,
|
218
|
-
:s3_bucket => user_s3_bucket, :
|
218
|
+
:s3_bucket => user_s3_bucket, :s3_file_key => user_s3_file_key}
|
219
219
|
s3_params[:cycle_id] = options[:cycle_id] if options[:cycle_id]
|
220
220
|
|
221
221
|
res = self.api.data.pull_to_s3(id, s3_params)
|
@@ -229,9 +229,9 @@ class Zillabyte::Command::Relations < Zillabyte::Command::Base
|
|
229
229
|
|
230
230
|
# relations:show ID
|
231
231
|
#
|
232
|
-
# shows
|
232
|
+
# shows a sample of the data in a relation. see 'zillabyte queries' for more elaborate functionality
|
233
233
|
#
|
234
|
-
# --cycle_id [cycle_id] #
|
234
|
+
# --cycle_id [cycle_id] # retrieve data generated for that cycle, only if this relation is associated with a flow. (defaults to the last cycle)
|
235
235
|
# --no_truncation # don't truncate long strings
|
236
236
|
#
|
237
237
|
def show
|
@@ -1,22 +1,29 @@
|
|
1
1
|
require 'zillabyte'
|
2
2
|
|
3
|
-
Zillabyte.simple_function do
|
3
|
+
Zillabyte.simple_function do
|
4
|
+
|
5
|
+
# This specifies the function's name and is mandatory.
|
6
|
+
name "simple_function"
|
4
7
|
|
5
8
|
# This directive instructs zillabyte to give your function every
|
6
9
|
# web page in our known universe. Your function will have access
|
7
10
|
# to two fields: URL and HTML
|
8
|
-
|
11
|
+
matches "select * from web_pages"
|
9
12
|
|
10
13
|
# This directive tells Zillabyte what kind of data your function
|
11
14
|
# produces. In this case, we're saying we will emit a tuple that
|
12
15
|
# is one-column wide and contains the field 'URL'
|
13
|
-
|
16
|
+
emits [
|
17
|
+
[
|
18
|
+
"has_hello_world", [ {"URL"=>:string} ]
|
19
|
+
]
|
20
|
+
]
|
14
21
|
|
15
22
|
|
16
23
|
# This is the heart of your algorithm. It's processed on every
|
17
24
|
# web page. This algorithm is run in parallel on possibly hundreds
|
18
25
|
# of machines.
|
19
|
-
|
26
|
+
execute do |tuple|
|
20
27
|
|
21
28
|
# get the fields
|
22
29
|
url = tuple['url']
|
@@ -27,13 +34,9 @@ Zillabyte.simple_function do |fn|
|
|
27
34
|
# instead of
|
28
35
|
# if html.include?('hello world')
|
29
36
|
if html.include?('hello world')
|
30
|
-
|
37
|
+
emit("has_hello_world", "URL" => url)
|
31
38
|
end
|
32
39
|
|
33
40
|
end
|
34
41
|
|
35
42
|
end
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
data/zillabyte-cli.gemspec
CHANGED
@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
|
|
28
28
|
spec.add_dependency "excon", "~> 0.31"
|
29
29
|
spec.add_dependency "terminal-table", "~> 1.4"
|
30
30
|
spec.add_dependency "activesupport", "~> 3.2.11"
|
31
|
+
spec.add_dependency "multi_json", "~> 1.0"
|
32
|
+
spec.add_dependency "mini_portile", "~> 0.5.0"
|
31
33
|
|
32
34
|
spec.add_dependency "bundler", "~> 1.3"
|
33
35
|
spec.add_dependency "colorize", "~> 0.6"
|
@@ -36,4 +38,5 @@ Gem::Specification.new do |spec|
|
|
36
38
|
spec.add_dependency "indentation", "~> 0.1"
|
37
39
|
spec.add_dependency "aws-sdk", "~> 1.33.0"
|
38
40
|
spec.add_dependency "time_difference"
|
41
|
+
|
39
42
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: zillabyte-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- zillabyte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-04-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -94,6 +94,34 @@ dependencies:
|
|
94
94
|
- - ~>
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: 3.2.11
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: multi_json
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ~>
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '1.0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ~>
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '1.0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: mini_portile
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ~>
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 0.5.0
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ~>
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 0.5.0
|
97
125
|
- !ruby/object:Gem::Dependency
|
98
126
|
name: bundler
|
99
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -201,7 +229,6 @@ extensions: []
|
|
201
229
|
extra_rdoc_files: []
|
202
230
|
files:
|
203
231
|
- bin/zillabyte
|
204
|
-
- lib/zillabyte/#api.rb#
|
205
232
|
- lib/zillabyte/api/base.rb
|
206
233
|
- lib/zillabyte/api/data.rb
|
207
234
|
- lib/zillabyte/api/flows.rb
|
@@ -215,7 +242,6 @@ files:
|
|
215
242
|
- lib/zillabyte/api/zillalogs.rb
|
216
243
|
- lib/zillabyte/api.rb
|
217
244
|
- lib/zillabyte/auth.rb
|
218
|
-
- lib/zillabyte/cli/#logs.rb#
|
219
245
|
- lib/zillabyte/cli/auth.rb
|
220
246
|
- lib/zillabyte/cli/base.rb
|
221
247
|
- lib/zillabyte/cli/config.rb
|
@@ -224,12 +250,12 @@ files:
|
|
224
250
|
- lib/zillabyte/cli/help.rb
|
225
251
|
- lib/zillabyte/cli/helpers/data_schema_builder.rb
|
226
252
|
- lib/zillabyte/cli/host.rb
|
253
|
+
- lib/zillabyte/cli/log_formatter.rb
|
227
254
|
- lib/zillabyte/cli/query.rb
|
228
255
|
- lib/zillabyte/cli/relations.rb
|
229
256
|
- lib/zillabyte/cli/sources.rb
|
230
257
|
- lib/zillabyte/cli/templates/js/simple_function.js
|
231
258
|
- lib/zillabyte/cli/templates/js/zillabyte.conf.yaml
|
232
|
-
- lib/zillabyte/cli/templates/python/#simple_function.py#
|
233
259
|
- lib/zillabyte/cli/templates/python/requirements.txt
|
234
260
|
- lib/zillabyte/cli/templates/python/simple_function.py
|
235
261
|
- lib/zillabyte/cli/templates/python/zillabyte.conf.yaml
|
@@ -273,7 +299,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
273
299
|
version: '0'
|
274
300
|
requirements: []
|
275
301
|
rubyforge_project:
|
276
|
-
rubygems_version: 2.
|
302
|
+
rubygems_version: 2.0.7
|
277
303
|
signing_key:
|
278
304
|
specification_version: 4
|
279
305
|
summary: The Official Zillabyte CLI Gem
|
data/lib/zillabyte/#api.rb#
DELETED
@@ -1,170 +0,0 @@
|
|
1
|
-
# load('zillabyte/helpers.rb') # reload helpers after possible inject_loadpath
|
2
|
-
# load('zillabyte/updater.rb') # reload updater after possible inject_loadpath
|
3
|
-
|
4
|
-
require "base64"
|
5
|
-
require "excon"
|
6
|
-
# require "securerandom"
|
7
|
-
require "uri"
|
8
|
-
require "zlib"
|
9
|
-
require "active_support/core_ext/object/to_query.rb"
|
10
|
-
require 'json'
|
11
|
-
|
12
|
-
|
13
|
-
# workaround for rescue/reraise to define errors in command.rb failing in 1.8.6
|
14
|
-
if RUBY_VERSION =~ /^1.8.6/
|
15
|
-
# require('zillabyte-api')
|
16
|
-
require('rest_client')
|
17
|
-
end
|
18
|
-
|
19
|
-
class Zillabyte::API
|
20
|
-
|
21
|
-
HEADERS = {
|
22
|
-
'Accept' => 'application/json',
|
23
|
-
'Content-Type' => 'application/json',
|
24
|
-
# 'Accept-Encoding' => 'gzip',
|
25
|
-
'User-Agent' => "zillabyte/#{Zillabyte::CLI::VERSION}",
|
26
|
-
'X-Ruby-Version' => RUBY_VERSION,
|
27
|
-
'X-Ruby-Platform' => RUBY_PLATFORM
|
28
|
-
}
|
29
|
-
|
30
|
-
OPTIONS = {
|
31
|
-
:headers => {},
|
32
|
-
:host => ENV['ZILLABYTE_API_HOST'] || 'api.zillabyte.com',
|
33
|
-
:port => (ENV['ZILLABYTE_API_PORT'] || '80').to_i,
|
34
|
-
:nonblock => false,
|
35
|
-
:scheme => 'http'
|
36
|
-
}
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
def initialize(options={})
|
41
|
-
options = OPTIONS.merge(options)
|
42
|
-
|
43
|
-
@progress = options.delete(:progress)
|
44
|
-
@api_key = options.delete(:api_key) || ENV['ZILLABYTE_API_KEY']
|
45
|
-
|
46
|
-
options[:headers] = HEADERS.merge({
|
47
|
-
'Authorization' => "auth_token #{@api_key}",
|
48
|
-
}).merge(options[:headers])
|
49
|
-
|
50
|
-
@options = options.clone
|
51
|
-
@options[:bubble_exceptions] = ENV['BUBBLE_EXCEPTIONS'] || false
|
52
|
-
|
53
|
-
@connection = Excon.new("#{options[:scheme]}://#{options[:host]}", options)
|
54
|
-
|
55
|
-
end
|
56
|
-
|
57
|
-
|
58
|
-
def host
|
59
|
-
@options[:host]
|
60
|
-
end
|
61
|
-
|
62
|
-
|
63
|
-
def request(params, &block)
|
64
|
-
begin
|
65
|
-
# The @connection object handles connections to api.zillabyte.com via Excon, this sends a command to the Rails api code
|
66
|
-
# with parameters in params (one of which contains the path to the method to be run in the form of a url, e.g. if the
|
67
|
-
# :path parameter is set to /functions/#{id}?tar=1, then the show method in functions_controller.rb will be automatically
|
68
|
-
# called...it will have it's own set of parameters which correspond to the items in the query string).
|
69
|
-
response = @connection.request(params, &block)
|
70
|
-
rescue Excon::Errors::HTTPStatusError => error
|
71
|
-
@progress.error "internal server error" if @progress
|
72
|
-
raise error
|
73
|
-
end
|
74
|
-
|
75
|
-
if response.body && !response.body.empty?
|
76
|
-
# decompress_response!(response)
|
77
|
-
begin
|
78
|
-
response.body = JSON.parse(response.body)
|
79
|
-
rescue Exception => e
|
80
|
-
raise e if @options[:bubble_exceptions]
|
81
|
-
@progress.error "unknown server response: \n #{response.body}" if @progress
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
# Errors?
|
86
|
-
if response.body.is_a?(Hash)
|
87
|
-
if response.body["status"] == "error"
|
88
|
-
if @progress
|
89
|
-
if response.body["error"] == "authentication_error"
|
90
|
-
@progress.error "authentication error. run 'zillabyte login'"
|
91
|
-
else
|
92
|
-
@progress.error response.body["error_message"]
|
93
|
-
end
|
94
|
-
end
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
|
99
|
-
# reset (non-persistent) connection
|
100
|
-
@connection.reset
|
101
|
-
|
102
|
-
response
|
103
|
-
end
|
104
|
-
|
105
|
-
|
106
|
-
def data
|
107
|
-
@_data ||= ::Zillabyte::API::Data.new(self)
|
108
|
-
end
|
109
|
-
|
110
|
-
def logs
|
111
|
-
@_logs ||= ::Zillabyte::API::Logs.new(self)
|
112
|
-
end
|
113
|
-
|
114
|
-
|
115
|
-
def queries
|
116
|
-
@_queries ||= ::Zillabyte::API::Queries.new(self)
|
117
|
-
end
|
118
|
-
alias_method :query, :queries
|
119
|
-
|
120
|
-
def flows
|
121
|
-
@_flows ||= ::Zillabyte::API::Flows.new(self)
|
122
|
-
end
|
123
|
-
alias_method :flow, :flows
|
124
|
-
|
125
|
-
def metrics
|
126
|
-
@_metrics ||= ::Zillabyte::API::Metrics.new(self)
|
127
|
-
end
|
128
|
-
alias_method :metric, :metrics
|
129
|
-
|
130
|
-
def zillalogs
|
131
|
-
@_rules ||= ::Zillabyte::API::Zillalogs.new(self)
|
132
|
-
end
|
133
|
-
alias_method :zillalog, :zillalogs
|
134
|
-
|
135
|
-
def sources
|
136
|
-
@_sources ||= ::Zillabyte::API::Sources.new(self)
|
137
|
-
end
|
138
|
-
alias_method :source, :sources
|
139
|
-
|
140
|
-
|
141
|
-
def collectors
|
142
|
-
@_collectors ||= ::Zillabyte::API::Collectors.new(self)
|
143
|
-
end
|
144
|
-
alias_method :collector, :collectors
|
145
|
-
|
146
|
-
def semantic_tags
|
147
|
-
@_rules ||= ::Zillabyte::API::SemanticTags.new(self)
|
148
|
-
end
|
149
|
-
alias_method :zillalog, :zillalogs
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
def self.load
|
156
|
-
Dir[File.join(File.dirname(__FILE__), "api", "*.rb")].sort.each do |file|
|
157
|
-
require file
|
158
|
-
end
|
159
|
-
|
160
|
-
Dir[File.join(File.dirname(__FILE__), "api", "helpers", "*.rb")].sort.each do |file|
|
161
|
-
require file
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
|
166
|
-
end
|
167
|
-
|
168
|
-
|
169
|
-
# Load all helpers...
|
170
|
-
Zillabyte::API.load()
|
data/lib/zillabyte/cli/#logs.rb#
DELETED
@@ -1,27 +0,0 @@
|
|
1
|
-
import zillabyte
|
2
|
-
|
3
|
-
def prep(controller):
|
4
|
-
return
|
5
|
-
|
6
|
-
# This is the heart of your algorithm. It's processed on every
|
7
|
-
# web page. This algorithm is run in parallel on possibly hundreds
|
8
|
-
# of machines.
|
9
|
-
def execute(controller, tup):
|
10
|
-
if("hello world" in tup.values["html"]):
|
11
|
-
controller.emit("has_hello_world",{"url":tup.values["url"]})
|
12
|
-
return
|
13
|
-
|
14
|
-
zillabyte.simple_function(\
|
15
|
-
# This directive instructs zillabyte to give your function every
|
16
|
-
# web page in our known universe. Your function will have access
|
17
|
-
# to two fields: URL and HTML
|
18
|
-
matches = "select * from web_pa", \
|
19
|
-
|
20
|
-
# This directive tells Zillabyte what kind of data your function
|
21
|
-
# produces. In this case, we're saying we will emit a tuple that
|
22
|
-
# is one-column wide and contains the field 'URL'
|
23
|
-
emits = [["has_hello_world", [{"url":"string"}]]], \
|
24
|
-
|
25
|
-
prepare = prep, \
|
26
|
-
execute = execute\
|
27
|
-
)
|