zillabyte-cli 0.0.24 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +6 -14
- data/lib/#zillabyte-cli.rb# +5 -0
- data/lib/zillabyte/api/apps.rb +16 -132
- data/lib/zillabyte/api/components.rb +115 -0
- data/lib/zillabyte/api/flows.rb +121 -0
- data/lib/zillabyte/api/keys.rb +70 -0
- data/lib/zillabyte/api.rb +15 -2
- data/lib/zillabyte/auth.rb +43 -16
- data/lib/zillabyte/cli/#logs.rb# +12 -0
- data/lib/zillabyte/cli/#repl.rb# +43 -0
- data/lib/zillabyte/cli/apps.rb +52 -893
- data/lib/zillabyte/cli/auth.rb +3 -8
- data/lib/zillabyte/cli/base.rb +28 -7
- data/lib/zillabyte/cli/components.rb +245 -0
- data/lib/zillabyte/cli/flows.rb +549 -0
- data/lib/zillabyte/cli/git.rb +38 -0
- data/lib/zillabyte/cli/help.rb +11 -4
- data/lib/zillabyte/cli/keys.rb +177 -0
- data/lib/zillabyte/cli/query.rb +0 -1
- data/lib/zillabyte/cli/relations.rb +2 -1
- data/lib/zillabyte/cli/templates/{js → apps/js}/simple_function.js +0 -0
- data/lib/zillabyte/cli/templates/{js → apps/js}/zillabyte.conf.yaml +0 -0
- data/lib/zillabyte/cli/templates/apps/python/app.py +17 -0
- data/lib/zillabyte/cli/templates/{python → apps/python}/requirements.txt +0 -0
- data/lib/zillabyte/cli/templates/{python → apps/python}/zillabyte.conf.yaml +1 -1
- data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/Gemfile +0 -0
- data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/app.rb +1 -1
- data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/zillabyte.conf.yaml +0 -0
- data/lib/zillabyte/cli/templates/python/{simple_function.py → #simple_function.py#} +3 -6
- data/lib/zillabyte/common/session.rb +3 -1
- data/lib/zillabyte/helpers.rb +64 -1
- data/lib/zillabyte/runner/app_runner.rb +226 -0
- data/lib/zillabyte/runner/component_operation.rb +529 -0
- data/lib/zillabyte/runner/component_runner.rb +244 -0
- data/lib/zillabyte/runner/multilang_operation.rb +1133 -0
- data/lib/zillabyte/runner/operation.rb +11 -0
- data/lib/zillabyte/runner.rb +6 -0
- data/lib/zillabyte-cli/version.rb +1 -1
- data/zillabyte-cli.gemspec +1 -0
- metadata +83 -52
@@ -0,0 +1,244 @@
|
|
1
|
+
require "zillabyte/cli/base"
|
2
|
+
require "zillabyte/runner"
|
3
|
+
require "zillabyte/runner/multilang_operation"
|
4
|
+
require "zillabyte/runner/component_operation"
|
5
|
+
require 'thread'
|
6
|
+
|
7
|
+
# HIDDEN:
|
8
|
+
class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
|
9
|
+
include Zillabyte::Helpers
|
10
|
+
|
11
|
+
def run (dir = Dir.pwd, session = nil, options = {})
|
12
|
+
|
13
|
+
if session.nil?
|
14
|
+
return
|
15
|
+
end
|
16
|
+
|
17
|
+
@session = session
|
18
|
+
@colors = {}
|
19
|
+
|
20
|
+
# Get options
|
21
|
+
input = options[:input]
|
22
|
+
output = options[:output]
|
23
|
+
otype = options[:output_type]
|
24
|
+
|
25
|
+
# Get component metadata
|
26
|
+
meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
|
27
|
+
if meta.nil? || meta["nodes"].nil?
|
28
|
+
error "this is not a valid zillabyte app directory"
|
29
|
+
exit
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
# Check that multilang version is atleast 0.1.0
|
34
|
+
version = meta["multilang_version"] || "0.0.0"
|
35
|
+
version_arr = version.split('.').map {|v| v.to_i}
|
36
|
+
if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
|
37
|
+
display "The version of zillabyte used in your application is outdated."
|
38
|
+
display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
|
39
|
+
return
|
40
|
+
end
|
41
|
+
|
42
|
+
# Show the user what we know about their app...
|
43
|
+
display "inferring your app details..."
|
44
|
+
describe_component(meta)
|
45
|
+
|
46
|
+
# Setup streams
|
47
|
+
@nodes = meta["nodes"]
|
48
|
+
|
49
|
+
# Index stream consummers and emitters by stream name
|
50
|
+
@arcs = meta["arcs"]
|
51
|
+
|
52
|
+
# Organize component pipes
|
53
|
+
@operations = {}
|
54
|
+
@operation_pipes = {}
|
55
|
+
|
56
|
+
|
57
|
+
# Start component
|
58
|
+
begin
|
59
|
+
|
60
|
+
# Setup operation pipes
|
61
|
+
@nodes.each do |n|
|
62
|
+
|
63
|
+
name = n["name"]
|
64
|
+
type = n["type"]
|
65
|
+
if n["type"] == "source"
|
66
|
+
fields = n["fields"]
|
67
|
+
end
|
68
|
+
|
69
|
+
# Create two new pipes in the parent.
|
70
|
+
rd_child, wr_parent = IO.pipe()
|
71
|
+
rd_parent, wr_child = IO.pipe()
|
72
|
+
|
73
|
+
@operation_pipes[name] = {
|
74
|
+
:node => n,
|
75
|
+
:rd_child => rd_child,
|
76
|
+
:wr_child => wr_child,
|
77
|
+
:rd_parent => rd_parent,
|
78
|
+
:wr_parent => wr_parent
|
79
|
+
}
|
80
|
+
end
|
81
|
+
|
82
|
+
# Maps origin => {stream => [destinations]}
|
83
|
+
@arc_map = {}
|
84
|
+
@arcs.each do |a|
|
85
|
+
origin = a["origin"]
|
86
|
+
name = a["name"]
|
87
|
+
dest = a["dest"]
|
88
|
+
@arc_map[origin] ||= {}
|
89
|
+
@arc_map[origin][name] ||= []
|
90
|
+
@arc_map[origin][name] << a["dest"]
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
# Spawn component threads
|
95
|
+
@nodes.each do |n|
|
96
|
+
|
97
|
+
name = n["name"]
|
98
|
+
type = n["type"]
|
99
|
+
emits = n["emits"]
|
100
|
+
|
101
|
+
pipes = @operation_pipes[name]
|
102
|
+
rd_child = pipes[:rd_child]
|
103
|
+
wr_child = pipes[:wr_child]
|
104
|
+
rd_parent = pipes[:rd_parent]
|
105
|
+
wr_parent = pipes[:wr_parent]
|
106
|
+
|
107
|
+
# Fork.
|
108
|
+
pid = fork()
|
109
|
+
if pid # In parent
|
110
|
+
# Close the reading end of the child so we can write to the child.
|
111
|
+
rd_child.close()
|
112
|
+
# Close the writing end of the child so we can read from the child.
|
113
|
+
wr_child.close()
|
114
|
+
|
115
|
+
else # in child
|
116
|
+
# Close the writing end of the parent so we can read from the parent.
|
117
|
+
wr_parent.close()
|
118
|
+
# Close the reading end of the parent so we can write to the parent.
|
119
|
+
rd_parent.close()
|
120
|
+
begin
|
121
|
+
|
122
|
+
# Setup reading and writing pipes for communicating with consumee component
|
123
|
+
in_pipe = {:rd_child => @operation_pipes[name][:rd_child], :wr_child => @operation_pipes[name][:wr_child]}
|
124
|
+
|
125
|
+
# Index consumer pipes by consumer_name
|
126
|
+
out_pipes = {}
|
127
|
+
|
128
|
+
if type != "sink"
|
129
|
+
@arc_map[name].each_pair do |stream, destinations|
|
130
|
+
out_pipes[stream] ||= {}
|
131
|
+
destinations.each do |dest|
|
132
|
+
out_pipes[stream][dest] = {:wr_parent => @operation_pipes[dest][:wr_parent], :rd_parent => @operation_pipes[dest][:rd_parent] }
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
# Run the child process
|
138
|
+
Zillabyte::Runner::ComponentOperation.run(n, dir, in_pipe, out_pipes, self, meta, options)
|
139
|
+
|
140
|
+
rescue => e
|
141
|
+
display e
|
142
|
+
|
143
|
+
ensure
|
144
|
+
# Close the reading end of the child
|
145
|
+
rd_child.close()
|
146
|
+
# Close the writing end of the child
|
147
|
+
wr_child.close()
|
148
|
+
exit!(-1)
|
149
|
+
end
|
150
|
+
|
151
|
+
end #end child
|
152
|
+
end
|
153
|
+
|
154
|
+
|
155
|
+
# If no input file, read from STDIN
|
156
|
+
# TODO handle inputs
|
157
|
+
if input.nil?
|
158
|
+
|
159
|
+
|
160
|
+
display "Use Ctrl-C to exit"
|
161
|
+
while true
|
162
|
+
|
163
|
+
# TODO this doesnt handle multiple sources
|
164
|
+
#source = @operation_pipes["source_1"][:node]
|
165
|
+
|
166
|
+
display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
|
167
|
+
msg = ask
|
168
|
+
|
169
|
+
begin
|
170
|
+
JSON.parse(msg)
|
171
|
+
rescue JSON::ParserError
|
172
|
+
display "Received invalid JSON object"
|
173
|
+
next
|
174
|
+
end
|
175
|
+
# Send tuple to source
|
176
|
+
@operation_pipes["stream_1"][:wr_parent].puts msg
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
ensure
|
181
|
+
Process.waitall()
|
182
|
+
@operation_pipes.each do |name, pipes|
|
183
|
+
#Close the writing end of the parent
|
184
|
+
pipes[:wr_parent].close()
|
185
|
+
# Close the reading end of the parent
|
186
|
+
pipes[:rd_parent].close()
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
end
|
191
|
+
|
192
|
+
def session
|
193
|
+
@session
|
194
|
+
end
|
195
|
+
|
196
|
+
|
197
|
+
def cdisplay(name, message)
|
198
|
+
color = @colors[name] || :default
|
199
|
+
if message == ""
|
200
|
+
display ""
|
201
|
+
else
|
202
|
+
display "#{name} - #{message}".colorize(color)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
|
207
|
+
def display(message, newline = true)
|
208
|
+
@session.display(message, newline)
|
209
|
+
end
|
210
|
+
|
211
|
+
def describe_component(meta)
|
212
|
+
colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
|
213
|
+
rjust = 20
|
214
|
+
|
215
|
+
display "#{'app name'.rjust(rjust)}: #{meta['name']}"
|
216
|
+
display "#{'app language'.rjust(rjust)}: #{meta['language']}"
|
217
|
+
meta['nodes'].each_with_index do |node, index|
|
218
|
+
color = @colors[node['name']] ||= colors[index % colors.length]
|
219
|
+
display (("="*rjust + " operation ##{index}").colorize(color))
|
220
|
+
display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
|
221
|
+
|
222
|
+
# Convert metadata typing to that of components
|
223
|
+
if node['type'] == "source"
|
224
|
+
type = "input"
|
225
|
+
display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
|
226
|
+
display "#{"fields".rjust(rjust)}: #{node['fields'].to_s.colorize(color)}"
|
227
|
+
display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
|
228
|
+
elsif node['type'] == "sink"
|
229
|
+
type = "output"
|
230
|
+
display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
|
231
|
+
display "#{"columns".rjust(rjust)}: #{node['columns'].to_s.colorize(color)}"
|
232
|
+
|
233
|
+
else
|
234
|
+
type = node['type']
|
235
|
+
display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
|
236
|
+
end
|
237
|
+
end
|
238
|
+
end
|
239
|
+
|
240
|
+
|
241
|
+
|
242
|
+
|
243
|
+
|
244
|
+
end
|