emptyd 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d30169c9b6b2194aa9463c384bca30e153180adb
4
+ data.tar.gz: 1cbb92a9adb7026e0753b540bc48229547e26da0
5
+ SHA512:
6
+ metadata.gz: 863c278fa138f2a375a7324f385684c609c253acb478d59a8a642aa9340bf4720bef71b4829ed5f634c7d3307f53717e5cc1d5e1403be5e174ca34eb59b855ac
7
+ data.tar.gz: 721c1cfbcf09a778de8121062eef223292fc182d6f44c1f8eea7815a7e33f925ac747633de7d6ac41cb787323e0da7eea981b8339c6009025cbd4c2b9bee8bfd
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in emptyd.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 kmeaw
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ # Emptyd
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'emptyd'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install emptyd
18
+
19
+ ## Usage
20
+
21
+ TODO: Write usage instructions here
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
data/bin/emptyd ADDED
@@ -0,0 +1,77 @@
1
+ #!/usr/bin/env ruby
2
+ # vim: ai:ts=2:sw=2:et:syntax=ruby
3
+
4
+ lp = File.expand_path(File.join(File.dirname(__FILE__), "..", "lib"))
5
+
6
+ unless $LOAD_PATH.include?(lp)
7
+ $LOAD_PATH.unshift(lp)
8
+ end
9
+
10
+ require "emptyd"
11
+ require 'evma_httpserver'
12
+
13
+ class MyHttpServer < EM::Connection
14
+ include EM::HttpServer
15
+
16
+ def post_init
17
+ super
18
+ no_environment_strings
19
+ end
20
+
21
+ def process_http_request
22
+ done = false
23
+ port, ip = Socket.unpack_sockaddr_in(get_peername)
24
+ Emptyd::LOG.info "#{ip}:#{port} #{@http_request_method} #{@http_path_info} #{@http_post_content.inspect}"
25
+
26
+ response = EM::DelegatedHttpResponse.new(self)
27
+ response.status = 200
28
+ response.content_type 'application/json'
29
+ response.content = JSON.dump({ :okay => true })
30
+
31
+ begin
32
+ case @http_path_info
33
+ when %r{/session/new}
34
+ session = Emptyd::Session.new JSON.load(@http_post_content)
35
+ response.content = JSON.dump({id: session.uuid})
36
+ when %r{/session/(.*)/run}
37
+ Emptyd::Session[$1].run @http_post_content
38
+ when %r{/session/(.*)/read}
39
+ session = Emptyd::Session[$1]
40
+ if session.dead? and session.queue.empty?
41
+ session.destroy!
42
+ response.content = JSON.dump([nil,:dead,nil])
43
+ else
44
+ return session.queue.pop do |data|
45
+ response.content = JSON.dump(data)
46
+ response.send_response
47
+ end
48
+ end
49
+ when %r{/session/(.*)/terminate}
50
+ session = Emptyd::Session[$1]
51
+ session.terminate @http_post_content
52
+ when %r{/session/(.*)}
53
+ session = Emptyd::Session[$1]
54
+ case @http_request_method
55
+ when "GET"
56
+ response.content = JSON.dump(session.status)
57
+ when "DELETE"
58
+ session.destroy
59
+ end
60
+ when %r{/session}
61
+ response.content = JSON.dump(Emptyd::Session.ids)
62
+ else
63
+ raise KeyError
64
+ end
65
+ rescue KeyError
66
+ response.content = JSON.dump({:okay => false, :error => :not_found})
67
+ response.status = 404
68
+ end
69
+ response.send_response
70
+ end
71
+ end
72
+
73
+ EM.run do
74
+ EM.start_server '0.0.0.0', 8080, MyHttpServer
75
+ Emptyd::LOG.info "Server started on port 8080."
76
+ end
77
+
data/emptyd.gemspec ADDED
@@ -0,0 +1,26 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'emptyd/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "emptyd"
8
+ spec.version = Emptyd::VERSION
9
+ spec.authors = ["kmeaw"]
10
+ spec.email = ["kmeaw@larkit.ru"]
11
+ spec.description = %q{An HTTP interface to run a single command on a cluster.}
12
+ spec.summary = %q{Run commands on multiple hosts over SSH}
13
+ spec.homepage = "https://github.com/kmeaw/emptyd"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files`.split($/)
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_development_dependency "bundler", "~> 1.3"
22
+ spec.add_development_dependency "rake"
23
+
24
+ spec.add_dependency "em-ssh"
25
+ spec.add_dependency "eventmachine_httpserver"
26
+ end
@@ -0,0 +1,3 @@
1
+ module Emptyd
2
+ VERSION = "0.0.1"
3
+ end
data/lib/emptyd.rb ADDED
@@ -0,0 +1,320 @@
1
+ # vim: ai:ts=2:sw=2:et:syntax=ruby
2
+ require "emptyd/version"
3
+
4
+ require 'fiber'
5
+ require 'em-ssh'
6
+ require 'securerandom'
7
+ require 'json'
8
+
9
+ module Emptyd
10
+ $0 = "emptyd"
11
+ LOG = Logger.new(STDERR)
12
+
13
+ class Connection
14
+ attr_reader :key, :updated_at, :failed_at, :error
15
+ EXPIRE_INTERVAL = 600 # 10min
16
+ MAX_CONNECTIONS = 30
17
+ HAPPY_RATIO = 0.5
18
+ @@connections = {}
19
+ @@count = {}
20
+
21
+ def self.[](key)
22
+ @@connections[key] or Connection.new(key)
23
+ end
24
+
25
+ def initialize(key)
26
+ raise IOError, "already registered" if @@connections[key]
27
+ @key = key
28
+ @sessions = []
29
+ @user, @host = key.split('@', 2)
30
+ @user, @host = "root", @user if @host.nil?
31
+ @@connections[key] = self
32
+ @run_queue = []
33
+ self.start
34
+ @timer = EM::PeriodicTimer.new(rand(5..15)) do
35
+ start
36
+ destroy if old? and free?
37
+ end
38
+ end
39
+
40
+ def destroy
41
+ raise IOError, "sessions are still alive" unless @sessions.empty?
42
+ @timer.cancel
43
+ @start_timer.cancel if @start_timer
44
+ @@connections.delete @key
45
+ if @conn
46
+ @@count.delete self.key
47
+ conn = @conn
48
+ @conn = nil
49
+ Fiber.new do
50
+ conn.close
51
+ end.resume
52
+ end
53
+ LOG.debug "Destroying connection #{@key}"
54
+ end
55
+
56
+ def start
57
+ return if @conn or @connecting
58
+ @connecting = true
59
+
60
+ pressure = proc do
61
+ if @@count.size >= MAX_CONNECTIONS # pressure
62
+ c = @@count.select{|k,c| c.free?}.values.sample
63
+ if c
64
+ c.destroy
65
+ else
66
+ LOG.debug "pressure: no free connections: #{@@count.keys}"
67
+ end
68
+ end
69
+ end
70
+
71
+ starter = proc do
72
+ begin
73
+ pressure[]
74
+ if @@count.size >= MAX_CONNECTIONS
75
+ LOG.debug "Quota exceeded by #{@key}: #{@@count.size}"
76
+ @start_timer = EM::PeriodicTimer.new(rand(1..10)) do
77
+ pressure[]
78
+ if @@count.size < MAX_CONNECTIONS
79
+ @start_timer.cancel
80
+ EM.next_tick starter
81
+ else
82
+ LOG.debug "No more connection quota, deferring #{@key}..."
83
+ end
84
+ end
85
+ else
86
+ @@count[self.key] = self
87
+ LOG.debug "Created new conn: #{key}, quota = #{@@count.size}"
88
+ EM::Ssh.start(@host, @user, user_known_hosts_file: []) do |conn|
89
+ conn.errback do |err|
90
+ @@count.delete self.key
91
+ @conn = nil
92
+ STDERR.puts "Connection to #{@key} is broken: #{err}"
93
+ @error = err
94
+ @failed_at = Time.now
95
+ @connecting = false
96
+ @run_queue.each do |cmd,session,callback|
97
+ callback.call self, :error
98
+ end
99
+ end
100
+ conn.callback do |ssh|
101
+ @conn = ssh
102
+ @error = nil
103
+ @failed_at = nil
104
+ @updated_at = Time.now
105
+ @connecting = false
106
+ @run_queue.each do |cmd,session,callback|
107
+ if session.dead?
108
+ LOG.debug "Dropping pending run request from a dead session"
109
+ @error = "session is dead"
110
+ else
111
+ EM.next_tick { run cmd, session, &callback }
112
+ end
113
+ end
114
+ @run_queue.clear
115
+ if @error
116
+ ssh.close
117
+ @@count.delete self.key
118
+ end
119
+ end
120
+ end
121
+ end
122
+ rescue EventMachine::ConnectionError => e
123
+ @@count.delete self.key
124
+ @conn = nil
125
+ @error = e
126
+ @failed_at = Time.now
127
+ @connecting = false
128
+ @run_queue.each do |cmd,session,callback|
129
+ callback.call self, :error
130
+ end
131
+ end
132
+ end
133
+
134
+ EM.next_tick starter
135
+ end
136
+
137
+ def bind(session)
138
+ raise IOError, "already bound" if @sessions.include? session
139
+ @sessions << session
140
+ end
141
+
142
+ def unbind(session)
143
+ raise IOError, "not bound" unless @sessions.include? session
144
+ @run_queue.delete_if{|cmd,sess,cb| sess == session}
145
+ @sessions.delete session
146
+ end
147
+
148
+ def run cmd, session, &callback
149
+ unless @conn
150
+ @run_queue << [cmd, session, callback]
151
+ return
152
+ end
153
+
154
+ @conn.open_channel do |ch|
155
+ ch.exec cmd do |ch, success|
156
+ callback.call self, :init, ch
157
+
158
+ STDERR.puts "#{h}: exec failed: #{cmd}" unless success
159
+ ch.on_data do |c, data|
160
+ EM.next_tick do
161
+ @updated_at = Time.now
162
+ session.queue.push [@key,nil,data]
163
+ end
164
+ end
165
+
166
+ ch.on_extended_data do |c, type, data|
167
+ EM.next_tick do
168
+ @updated_at = Time.now
169
+ session.queue.push [@key,type,data]
170
+ LOG.debug [type,data]
171
+ end
172
+ end
173
+
174
+ ch.on_request "exit-status" do |ch, data|
175
+ EM.next_tick do
176
+ @updated_at = Time.now
177
+ session.queue.push [@key,:exit,data.read_long]
178
+ end
179
+ end
180
+
181
+ ch.on_close do
182
+ @updated_at = Time.now
183
+ callback.call self, :close
184
+ end
185
+ end
186
+ end
187
+ end
188
+
189
+ def old?
190
+ @updated_at and Time.now - @updated_at > EXPIRE_INTERVAL
191
+ end
192
+
193
+ def free?
194
+ @sessions.empty?
195
+ end
196
+
197
+ def connecting?
198
+ @connecting
199
+ end
200
+
201
+ def dead?
202
+ @failed_at
203
+ end
204
+ end
205
+
206
+ class Session
207
+ attr_reader :uuid, :queue
208
+ @@sessions = {}
209
+
210
+ def self.ids
211
+ @@sessions.keys
212
+ end
213
+
214
+ def self.[](uuid)
215
+ @@sessions[uuid] or raise KeyError, "no such session"
216
+ end
217
+
218
+ def initialize keys, &callback
219
+ @uuid = SecureRandom.uuid
220
+ @@sessions[@uuid] = self
221
+ @keys = keys
222
+ @connections = Hash[keys.map{|h| [h, Connection[h]]}]
223
+ @connections.each_value{|h| h.bind self}
224
+ @queue = EM::Queue.new
225
+ @running = {}
226
+ @dead = false
227
+ @terminated = {}
228
+ end
229
+
230
+ def destroy
231
+ LOG.debug "Destroying session #{@uuid}"
232
+ p @running.map{|h,v| [h, v.class.name]}
233
+ @running.each do |h,v|
234
+ if v.respond_to? :close
235
+ p "Closing channel for #{h}"
236
+ v.close
237
+ end
238
+ end
239
+ @connections.each_value do |h|
240
+ callback h, :close, "user"
241
+ h.unbind self
242
+ end
243
+ @dead = true
244
+ end
245
+
246
+ def destroy!
247
+ @@sessions.delete @uuid
248
+ end
249
+
250
+ def done?
251
+ @running.empty?
252
+ end
253
+
254
+ def dead?
255
+ @dead
256
+ end
257
+
258
+ def run cmd
259
+ dead = @connections.values.select(&:dead?)
260
+ alive = @connections.values.reject(&:dead?)
261
+ @queue.push [nil,:dead,dead.map(&:key)]
262
+ alive.each { |h| @running[h.key] = true }
263
+ alive.each do |h|
264
+ h.run(cmd, self) { |h,e,c| callback h,e,c }
265
+ end
266
+ end
267
+
268
+ def status
269
+ {
270
+ :children => Hash[@keys.map{|k| [k,
271
+ @terminated[k] ? :terminated :
272
+ @running[k] == true ? :pending :
273
+ @running[k] ? :running :
274
+ @connections[k] ?
275
+ @connections[k].dead? ? :dead : :unknown
276
+ : :done]}],
277
+ :dead => @dead
278
+ }
279
+ end
280
+
281
+ def terminate key
282
+ chan = @running[key]
283
+ conn = @connections[key]
284
+ chan.close if chan.respond_to? :close
285
+ if conn
286
+ callback conn, :close, "user"
287
+ conn.unbind self
288
+ end
289
+ @running.delete key
290
+ @connections.delete key
291
+ @terminated[key] = true
292
+ end
293
+
294
+ def callback(h,e,c=nil)
295
+ EM.next_tick do
296
+ LOG.debug [h.key,e,c.nil?]
297
+ case e
298
+ when :init
299
+ @queue.push [h.key,:start,nil]
300
+ @running[h.key] = c
301
+ when :close, :error
302
+ h.unbind self unless @dead or not @connections.include? h.key
303
+ @connections.delete h.key
304
+ @queue.push [h.key,:dead,nil] if e == :error
305
+ @queue.push [h.key,:done,nil]
306
+ @running.delete h.key
307
+ if done?
308
+ @queue.push [nil,:done,nil]
309
+ LOG.debug "run is done."
310
+ else
311
+ LOG.debug "#{@running.size} connections pending"
312
+ end
313
+ else
314
+ LOG.error "Session#run: unexpected callback #{e}"
315
+ end
316
+ end
317
+ end
318
+ end
319
+ end
320
+
metadata ADDED
@@ -0,0 +1,110 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: emptyd
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - kmeaw
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-03-14 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.3'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '>='
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: em-ssh
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: eventmachine_httpserver
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '>='
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ description: An HTTP interface to run a single command on a cluster.
70
+ email:
71
+ - kmeaw@larkit.ru
72
+ executables:
73
+ - emptyd
74
+ extensions: []
75
+ extra_rdoc_files: []
76
+ files:
77
+ - .gitignore
78
+ - Gemfile
79
+ - LICENSE.txt
80
+ - README.md
81
+ - Rakefile
82
+ - bin/emptyd
83
+ - emptyd.gemspec
84
+ - lib/emptyd.rb
85
+ - lib/emptyd/version.rb
86
+ homepage: https://github.com/kmeaw/emptyd
87
+ licenses:
88
+ - MIT
89
+ metadata: {}
90
+ post_install_message:
91
+ rdoc_options: []
92
+ require_paths:
93
+ - lib
94
+ required_ruby_version: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - '>='
97
+ - !ruby/object:Gem::Version
98
+ version: '0'
99
+ required_rubygems_version: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - '>='
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ requirements: []
105
+ rubyforge_project:
106
+ rubygems_version: 2.1.9
107
+ signing_key:
108
+ specification_version: 4
109
+ summary: Run commands on multiple hosts over SSH
110
+ test_files: []