zillabyte-cli 0.0.24 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +6 -14
  2. data/lib/#zillabyte-cli.rb# +5 -0
  3. data/lib/zillabyte/api/apps.rb +16 -132
  4. data/lib/zillabyte/api/components.rb +115 -0
  5. data/lib/zillabyte/api/flows.rb +121 -0
  6. data/lib/zillabyte/api/keys.rb +70 -0
  7. data/lib/zillabyte/api.rb +15 -2
  8. data/lib/zillabyte/auth.rb +43 -16
  9. data/lib/zillabyte/cli/#logs.rb# +12 -0
  10. data/lib/zillabyte/cli/#repl.rb# +43 -0
  11. data/lib/zillabyte/cli/apps.rb +52 -893
  12. data/lib/zillabyte/cli/auth.rb +3 -8
  13. data/lib/zillabyte/cli/base.rb +28 -7
  14. data/lib/zillabyte/cli/components.rb +245 -0
  15. data/lib/zillabyte/cli/flows.rb +549 -0
  16. data/lib/zillabyte/cli/git.rb +38 -0
  17. data/lib/zillabyte/cli/help.rb +11 -4
  18. data/lib/zillabyte/cli/keys.rb +177 -0
  19. data/lib/zillabyte/cli/query.rb +0 -1
  20. data/lib/zillabyte/cli/relations.rb +2 -1
  21. data/lib/zillabyte/cli/templates/{js → apps/js}/simple_function.js +0 -0
  22. data/lib/zillabyte/cli/templates/{js → apps/js}/zillabyte.conf.yaml +0 -0
  23. data/lib/zillabyte/cli/templates/apps/python/app.py +17 -0
  24. data/lib/zillabyte/cli/templates/{python → apps/python}/requirements.txt +0 -0
  25. data/lib/zillabyte/cli/templates/{python → apps/python}/zillabyte.conf.yaml +1 -1
  26. data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/Gemfile +0 -0
  27. data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/app.rb +1 -1
  28. data/lib/zillabyte/cli/templates/{ruby → apps/ruby}/zillabyte.conf.yaml +0 -0
  29. data/lib/zillabyte/cli/templates/python/{simple_function.py → #simple_function.py#} +3 -6
  30. data/lib/zillabyte/common/session.rb +3 -1
  31. data/lib/zillabyte/helpers.rb +64 -1
  32. data/lib/zillabyte/runner/app_runner.rb +226 -0
  33. data/lib/zillabyte/runner/component_operation.rb +529 -0
  34. data/lib/zillabyte/runner/component_runner.rb +244 -0
  35. data/lib/zillabyte/runner/multilang_operation.rb +1133 -0
  36. data/lib/zillabyte/runner/operation.rb +11 -0
  37. data/lib/zillabyte/runner.rb +6 -0
  38. data/lib/zillabyte-cli/version.rb +1 -1
  39. data/zillabyte-cli.gemspec +1 -0
  40. metadata +83 -52
@@ -0,0 +1,244 @@
1
+ require "zillabyte/cli/base"
2
+ require "zillabyte/runner"
3
+ require "zillabyte/runner/multilang_operation"
4
+ require "zillabyte/runner/component_operation"
5
+ require 'thread'
6
+
7
+ # HIDDEN:
8
+ class Zillabyte::Runner::ComponentRunner < Zillabyte::Command::Base
9
+ include Zillabyte::Helpers
10
+
11
+ def run (dir = Dir.pwd, session = nil, options = {})
12
+
13
+ if session.nil?
14
+ return
15
+ end
16
+
17
+ @session = session
18
+ @colors = {}
19
+
20
+ # Get options
21
+ input = options[:input]
22
+ output = options[:output]
23
+ otype = options[:output_type]
24
+
25
+ # Get component metadata
26
+ meta = Zillabyte::API::Flows.get_rich_meta_info_from_script(dir, @session, {:test => true})
27
+ if meta.nil? || meta["nodes"].nil?
28
+ error "this is not a valid zillabyte app directory"
29
+ exit
30
+ end
31
+
32
+
33
+ # Check that multilang version is atleast 0.1.0
34
+ version = meta["multilang_version"] || "0.0.0"
35
+ version_arr = version.split('.').map {|v| v.to_i}
36
+ if version_arr.empty? || (version_arr[0] == 0 && version_arr[1] < 1)
37
+ display "The version of zillabyte used in your application is outdated."
38
+ display "Please use upgrade your zillabyte gem via 'bundle update zillabyte; gem cleanup zillabyte'"
39
+ return
40
+ end
41
+
42
+ # Show the user what we know about their app...
43
+ display "inferring your app details..."
44
+ describe_component(meta)
45
+
46
+ # Setup streams
47
+ @nodes = meta["nodes"]
48
+
49
+ # Index stream consummers and emitters by stream name
50
+ @arcs = meta["arcs"]
51
+
52
+ # Organize component pipes
53
+ @operations = {}
54
+ @operation_pipes = {}
55
+
56
+
57
+ # Start component
58
+ begin
59
+
60
+ # Setup operation pipes
61
+ @nodes.each do |n|
62
+
63
+ name = n["name"]
64
+ type = n["type"]
65
+ if n["type"] == "source"
66
+ fields = n["fields"]
67
+ end
68
+
69
+ # Create two new pipes in the parent.
70
+ rd_child, wr_parent = IO.pipe()
71
+ rd_parent, wr_child = IO.pipe()
72
+
73
+ @operation_pipes[name] = {
74
+ :node => n,
75
+ :rd_child => rd_child,
76
+ :wr_child => wr_child,
77
+ :rd_parent => rd_parent,
78
+ :wr_parent => wr_parent
79
+ }
80
+ end
81
+
82
+ # Maps origin => {stream => [destinations]}
83
+ @arc_map = {}
84
+ @arcs.each do |a|
85
+ origin = a["origin"]
86
+ name = a["name"]
87
+ dest = a["dest"]
88
+ @arc_map[origin] ||= {}
89
+ @arc_map[origin][name] ||= []
90
+ @arc_map[origin][name] << a["dest"]
91
+ end
92
+
93
+
94
+ # Spawn component threads
95
+ @nodes.each do |n|
96
+
97
+ name = n["name"]
98
+ type = n["type"]
99
+ emits = n["emits"]
100
+
101
+ pipes = @operation_pipes[name]
102
+ rd_child = pipes[:rd_child]
103
+ wr_child = pipes[:wr_child]
104
+ rd_parent = pipes[:rd_parent]
105
+ wr_parent = pipes[:wr_parent]
106
+
107
+ # Fork.
108
+ pid = fork()
109
+ if pid # In parent
110
+ # Close the reading end of the child so we can write to the child.
111
+ rd_child.close()
112
+ # Close the writing end of the child so we can read from the child.
113
+ wr_child.close()
114
+
115
+ else # in child
116
+ # Close the writing end of the parent so we can read from the parent.
117
+ wr_parent.close()
118
+ # Close the reading end of the parent so we can write to the parent.
119
+ rd_parent.close()
120
+ begin
121
+
122
+ # Setup reading and writing pipes for communicating with consumee component
123
+ in_pipe = {:rd_child => @operation_pipes[name][:rd_child], :wr_child => @operation_pipes[name][:wr_child]}
124
+
125
+ # Index consumer pipes by consumer_name
126
+ out_pipes = {}
127
+
128
+ if type != "sink"
129
+ @arc_map[name].each_pair do |stream, destinations|
130
+ out_pipes[stream] ||= {}
131
+ destinations.each do |dest|
132
+ out_pipes[stream][dest] = {:wr_parent => @operation_pipes[dest][:wr_parent], :rd_parent => @operation_pipes[dest][:rd_parent] }
133
+ end
134
+ end
135
+ end
136
+
137
+ # Run the child process
138
+ Zillabyte::Runner::ComponentOperation.run(n, dir, in_pipe, out_pipes, self, meta, options)
139
+
140
+ rescue => e
141
+ display e
142
+
143
+ ensure
144
+ # Close the reading end of the child
145
+ rd_child.close()
146
+ # Close the writing end of the child
147
+ wr_child.close()
148
+ exit!(-1)
149
+ end
150
+
151
+ end #end child
152
+ end
153
+
154
+
155
+ # If no input file, read from STDIN
156
+ # TODO handle inputs
157
+ if input.nil?
158
+
159
+
160
+ display "Use Ctrl-C to exit"
161
+ while true
162
+
163
+ # TODO this doesnt handle multiple sources
164
+ #source = @operation_pipes["source_1"][:node]
165
+
166
+ display "Enter an input tuple in JSON format i.e.{ \"url\" : \"foo.com\", \"html\" : \"bar.html\" }"
167
+ msg = ask
168
+
169
+ begin
170
+ JSON.parse(msg)
171
+ rescue JSON::ParserError
172
+ display "Received invalid JSON object"
173
+ next
174
+ end
175
+ # Send tuple to source
176
+ @operation_pipes["stream_1"][:wr_parent].puts msg
177
+ end
178
+ end
179
+
180
+ ensure
181
+ Process.waitall()
182
+ @operation_pipes.each do |name, pipes|
183
+ #Close the writing end of the parent
184
+ pipes[:wr_parent].close()
185
+ # Close the reading end of the parent
186
+ pipes[:rd_parent].close()
187
+ end
188
+ end
189
+
190
+ end
191
+
192
+ def session
193
+ @session
194
+ end
195
+
196
+
197
+ def cdisplay(name, message)
198
+ color = @colors[name] || :default
199
+ if message == ""
200
+ display ""
201
+ else
202
+ display "#{name} - #{message}".colorize(color)
203
+ end
204
+ end
205
+
206
+
207
+ def display(message, newline = true)
208
+ @session.display(message, newline)
209
+ end
210
+
211
+ def describe_component(meta)
212
+ colors ||= [:green, :yellow, :magenta, :cyan, :white, :blue, :light_yellow, :light_blue, :red, :light_magenta, :light_cyan]
213
+ rjust = 20
214
+
215
+ display "#{'app name'.rjust(rjust)}: #{meta['name']}"
216
+ display "#{'app language'.rjust(rjust)}: #{meta['language']}"
217
+ meta['nodes'].each_with_index do |node, index|
218
+ color = @colors[node['name']] ||= colors[index % colors.length]
219
+ display (("="*rjust + " operation ##{index}").colorize(color))
220
+ display "#{"name".rjust(rjust)}: #{node['name'].to_s.colorize(color)}"
221
+
222
+ # Convert metadata typing to that of components
223
+ if node['type'] == "source"
224
+ type = "input"
225
+ display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
226
+ display "#{"fields".rjust(rjust)}: #{node['fields'].to_s.colorize(color)}"
227
+ display "#{"matches".rjust(rjust)}: #{JSON.pretty_generate(node['matches']).indent(rjust+2).lstrip.colorize(color)}" if node['matches']
228
+ elsif node['type'] == "sink"
229
+ type = "output"
230
+ display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
231
+ display "#{"columns".rjust(rjust)}: #{node['columns'].to_s.colorize(color)}"
232
+
233
+ else
234
+ type = node['type']
235
+ display "#{"type".rjust(rjust)}: #{type.to_s.colorize(color)}"
236
+ end
237
+ end
238
+ end
239
+
240
+
241
+
242
+
243
+
244
+ end