vines-services 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +19 -0
- data/README +40 -0
- data/Rakefile +130 -0
- data/bin/vines-services +95 -0
- data/conf/config.rb +25 -0
- data/lib/vines/services/command/init.rb +209 -0
- data/lib/vines/services/command/restart.rb +14 -0
- data/lib/vines/services/command/start.rb +30 -0
- data/lib/vines/services/command/stop.rb +20 -0
- data/lib/vines/services/command/views.rb +26 -0
- data/lib/vines/services/component.rb +26 -0
- data/lib/vines/services/config.rb +105 -0
- data/lib/vines/services/connection.rb +120 -0
- data/lib/vines/services/controller/attributes_controller.rb +19 -0
- data/lib/vines/services/controller/base_controller.rb +99 -0
- data/lib/vines/services/controller/disco_info_controller.rb +61 -0
- data/lib/vines/services/controller/labels_controller.rb +17 -0
- data/lib/vines/services/controller/members_controller.rb +44 -0
- data/lib/vines/services/controller/messages_controller.rb +66 -0
- data/lib/vines/services/controller/probes_controller.rb +45 -0
- data/lib/vines/services/controller/services_controller.rb +81 -0
- data/lib/vines/services/controller/subscriptions_controller.rb +39 -0
- data/lib/vines/services/controller/systems_controller.rb +45 -0
- data/lib/vines/services/controller/transfers_controller.rb +58 -0
- data/lib/vines/services/controller/uploads_controller.rb +62 -0
- data/lib/vines/services/controller/users_controller.rb +127 -0
- data/lib/vines/services/core_ext/blather.rb +46 -0
- data/lib/vines/services/core_ext/couchrest.rb +33 -0
- data/lib/vines/services/indexer.rb +195 -0
- data/lib/vines/services/priority_queue.rb +94 -0
- data/lib/vines/services/roster.rb +70 -0
- data/lib/vines/services/storage/couchdb/fragment.rb +23 -0
- data/lib/vines/services/storage/couchdb/service.rb +170 -0
- data/lib/vines/services/storage/couchdb/system.rb +141 -0
- data/lib/vines/services/storage/couchdb/upload.rb +66 -0
- data/lib/vines/services/storage/couchdb/user.rb +137 -0
- data/lib/vines/services/storage/couchdb/vcard.rb +13 -0
- data/lib/vines/services/storage/couchdb.rb +157 -0
- data/lib/vines/services/storage.rb +33 -0
- data/lib/vines/services/throttle.rb +26 -0
- data/lib/vines/services/version.rb +7 -0
- data/lib/vines/services/vql/compiler.rb +94 -0
- data/lib/vines/services/vql/vql.citrus +115 -0
- data/lib/vines/services/vql/vql.rb +186 -0
- data/lib/vines/services.rb +71 -0
- data/test/config_test.rb +242 -0
- data/test/priority_queue_test.rb +23 -0
- data/test/storage/couchdb_test.rb +30 -0
- data/test/vql/compiler_test.rb +96 -0
- data/test/vql/vql_test.rb +233 -0
- data/web/coffeescripts/api.coffee +51 -0
- data/web/coffeescripts/commands.coffee +18 -0
- data/web/coffeescripts/files.coffee +315 -0
- data/web/coffeescripts/init.coffee +21 -0
- data/web/coffeescripts/services.coffee +356 -0
- data/web/coffeescripts/setup.coffee +503 -0
- data/web/coffeescripts/systems.coffee +371 -0
- data/web/images/default-service.png +0 -0
- data/web/images/linux.png +0 -0
- data/web/images/mac.png +0 -0
- data/web/images/run.png +0 -0
- data/web/images/windows.png +0 -0
- data/web/index.html +17 -0
- data/web/stylesheets/common.css +52 -0
- data/web/stylesheets/files.css +218 -0
- data/web/stylesheets/services.css +181 -0
- data/web/stylesheets/setup.css +117 -0
- data/web/stylesheets/systems.css +142 -0
- metadata +230 -0
@@ -0,0 +1,39 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
module Controller
|
6
|
+
# Presence subscription requests are approved if the user has privilege to
|
7
|
+
# access the requested service JID. All subscriptions to the component
|
8
|
+
# itself are approved.
|
9
|
+
class SubscriptionsController < BaseController
|
10
|
+
register :subscription, :request?
|
11
|
+
|
12
|
+
def process
|
13
|
+
if approved?
|
14
|
+
from, to = node.from.stripped, node.to.stripped
|
15
|
+
stream.write(node.approve!)
|
16
|
+
stream.write(available(to, from))
|
17
|
+
else
|
18
|
+
stream.write(node.refuse!)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def approved?
|
25
|
+
return true if to_component?
|
26
|
+
found = Service.find_by_jid(node.to)
|
27
|
+
found && found.user?(node.from.stripped)
|
28
|
+
end
|
29
|
+
|
30
|
+
def available(from, to)
|
31
|
+
Blather::Stanza::Presence::Status.new.tap do |stanza|
|
32
|
+
stanza.from = from
|
33
|
+
stanza.to = to
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
module Controller
|
6
|
+
class SystemsController < BaseController
|
7
|
+
register :iq, "/iq[@type='get' or @type='set']/ns:query",
|
8
|
+
'ns' => 'http://getvines.com/protocol/systems'
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def get
|
13
|
+
if name = node.elements.first['name']
|
14
|
+
forbidden! unless current_user.manage_systems? || from_system?(name)
|
15
|
+
send_doc(System.find_by_name(name))
|
16
|
+
else
|
17
|
+
forbidden! unless current_user.manage_systems?
|
18
|
+
send_result(rows: System.find_all)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
# Agents are allowed to save system data for their machine only. No
|
23
|
+
# other user may change a system's description.
|
24
|
+
def save
|
25
|
+
obj = JSON.parse(node.elements.first.content)
|
26
|
+
fqdn = obj['fqdn'].downcase
|
27
|
+
forbidden! unless from_system?(fqdn)
|
28
|
+
|
29
|
+
id = "system:#{fqdn}"
|
30
|
+
system = System.find(id) || System.new(id: id)
|
31
|
+
system.ohai = obj
|
32
|
+
system.save
|
33
|
+
storage.index(system)
|
34
|
+
System.notify_members(stream, node.from, [{'name' => fqdn}])
|
35
|
+
send_result
|
36
|
+
end
|
37
|
+
|
38
|
+
# Return true if a System user is requesting access to its own data.
|
39
|
+
def from_system?(fqdn)
|
40
|
+
current_user.system? && node.from.node.downcase == fqdn.downcase
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
module Controller
|
6
|
+
class TransfersController < BaseController
|
7
|
+
register :file_transfer
|
8
|
+
|
9
|
+
def process
|
10
|
+
forbidden! unless current_user.manage_files?
|
11
|
+
return unless node.to == stream.jid
|
12
|
+
name, size = node.si.file['name'], node.si.file['size'].to_i
|
13
|
+
log.debug("Receiving file: #{name}")
|
14
|
+
|
15
|
+
path = File.expand_path(name, uploads)
|
16
|
+
return unless path.start_with?(uploads)
|
17
|
+
|
18
|
+
log.debug("Saving file to #{path}")
|
19
|
+
File.delete(path) if File.exist?(path)
|
20
|
+
|
21
|
+
save_file_doc(name, size)
|
22
|
+
|
23
|
+
transfer = Blather::FileTransfer.new(stream, node)
|
24
|
+
transfer.allow_s5b = false
|
25
|
+
transfer.accept(FileUploader, path, size, storage)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def save_file_doc(name, size)
|
31
|
+
file = Upload.find_by_name(name) || Upload.new
|
32
|
+
file.name = name
|
33
|
+
file.size = size
|
34
|
+
file.save
|
35
|
+
end
|
36
|
+
|
37
|
+
module FileUploader
|
38
|
+
include Blather::FileTransfer::SimpleFileReceiver
|
39
|
+
|
40
|
+
def initialize(path, size, storage)
|
41
|
+
super(path, size)
|
42
|
+
@storage = storage
|
43
|
+
end
|
44
|
+
|
45
|
+
def unbind
|
46
|
+
super
|
47
|
+
return unless File.exist?(@path)
|
48
|
+
Fiber.new do
|
49
|
+
@storage.store_file(@path) do
|
50
|
+
File.delete(@path) rescue nil
|
51
|
+
end
|
52
|
+
end.resume
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
module Controller
|
6
|
+
class UploadsController < BaseController
|
7
|
+
register :iq, "/iq[@type='get' or @type='set']/ns:query",
|
8
|
+
'ns' => 'http://getvines.com/protocol/files'
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
def get
|
13
|
+
forbidden! unless current_user.manage_files?
|
14
|
+
id, name, label = %w[id name label].map {|a| node.elements.first[a] }
|
15
|
+
|
16
|
+
if id
|
17
|
+
send_doc(Upload.find(id))
|
18
|
+
elsif name
|
19
|
+
send_doc(Upload.find_by_name(name))
|
20
|
+
elsif label
|
21
|
+
send_result(rows: Upload.find_by_label(label))
|
22
|
+
else
|
23
|
+
send_result(rows: Upload.find_all)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# Uploaded files may have their names and labels updated, but not their
|
28
|
+
# size. New files are created by uploading them to the UploadsController,
|
29
|
+
# not by saving them here.
|
30
|
+
def save
|
31
|
+
forbidden! unless current_user.manage_files?
|
32
|
+
obj = JSON.parse(node.elements.first.content)
|
33
|
+
file = Upload.find(obj['id'])
|
34
|
+
|
35
|
+
unless file
|
36
|
+
send_error('item-not-found')
|
37
|
+
return
|
38
|
+
end
|
39
|
+
|
40
|
+
file.name = obj['name']
|
41
|
+
file.labels = obj['labels']
|
42
|
+
if file.valid?
|
43
|
+
file.save
|
44
|
+
send_doc(file)
|
45
|
+
else
|
46
|
+
send_error('not-acceptable')
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def delete
|
51
|
+
forbidden! unless current_user.manage_files?
|
52
|
+
if file = Upload.find(node.elements.first['id'])
|
53
|
+
file.destroy
|
54
|
+
send_result
|
55
|
+
else
|
56
|
+
send_error('item-not-found')
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,127 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
module Controller
|
6
|
+
class UsersController < BaseController
|
7
|
+
register :iq, "/iq[@type='get' or @type='set']/ns:query",
|
8
|
+
'ns' => 'http://getvines.com/protocol/users'
|
9
|
+
|
10
|
+
private
|
11
|
+
|
12
|
+
# Returns user information to the client without the password. Passwords,
|
13
|
+
# despite being securely hashed, must never be sent to client requests.
|
14
|
+
def get
|
15
|
+
if jid = node.elements.first['jid']
|
16
|
+
user = User.find_by_jid(jid)
|
17
|
+
forbidden! unless authorized?(user)
|
18
|
+
send_doc(user)
|
19
|
+
else
|
20
|
+
users = User.find_all.select {|user| authorized?(user) }
|
21
|
+
send_result(rows: users)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# Save the user to the database. The password needs to be bcrypted
|
26
|
+
# before it is stored. Empty passwords will not be saved.
|
27
|
+
def save
|
28
|
+
obj = JSON.parse(node.elements.first.content)
|
29
|
+
forbidden! unless authorized?(obj)
|
30
|
+
|
31
|
+
system = (obj['system'] == true)
|
32
|
+
jid, name, username, password1, password2 =
|
33
|
+
%w[jid name username password1 password2].map {|a| (obj[a] || '').strip }
|
34
|
+
|
35
|
+
raise 'jid required' if jid.empty? && username.empty?
|
36
|
+
|
37
|
+
user = if jid.empty? # new user
|
38
|
+
id = "user:%s" % Vines::JID.new(username, node.from.domain)
|
39
|
+
raise 'user already exists' if User.find(id)
|
40
|
+
User.new(id: id).tap do |u|
|
41
|
+
u.system = system
|
42
|
+
u.password = password1
|
43
|
+
end
|
44
|
+
else # existing user
|
45
|
+
User.get!("user:%s" % Vines::JID.new(jid)).tap do |u|
|
46
|
+
raise "record found, but is a #{u.system ? 'system' : 'user'}" unless u.system == system
|
47
|
+
if system
|
48
|
+
u.password = password1 unless password1.empty?
|
49
|
+
else # humans
|
50
|
+
u.change_password(password1, password2) unless password1.empty? || password2.empty?
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
unless system
|
56
|
+
user.name = name
|
57
|
+
if user.jid == current_user.jid && !current_user.manage_services
|
58
|
+
if obj['permissions']['systems'] == true
|
59
|
+
user.valid = false
|
60
|
+
send_error('not-acceptable')
|
61
|
+
end
|
62
|
+
end
|
63
|
+
user.permissions = obj['permissions']
|
64
|
+
end
|
65
|
+
|
66
|
+
if user.valid?
|
67
|
+
user.save
|
68
|
+
save_services(user, obj['services'] || [])
|
69
|
+
send_doc(user)
|
70
|
+
else
|
71
|
+
send_error('not-acceptable')
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# Return true if the user is allowed to view and save the system or
|
76
|
+
# user object. Users with no permissions still have the right to update
|
77
|
+
# their own user account.
|
78
|
+
def authorized?(user)
|
79
|
+
return false unless user
|
80
|
+
user = user.to_result if user.respond_to?(:to_result)
|
81
|
+
system = user['system'] || user[:system]
|
82
|
+
jid = user['jid'] || user[:jid]
|
83
|
+
return current_user.manage_systems? if system
|
84
|
+
current_user.manage_users? || current_user.jid == jid
|
85
|
+
end
|
86
|
+
|
87
|
+
def save_services(user, services)
|
88
|
+
return if user.system?
|
89
|
+
return unless user.permissions['manage_services']
|
90
|
+
current = user.services.map {|s| s.id }
|
91
|
+
members = []
|
92
|
+
|
93
|
+
# delete missing services
|
94
|
+
user.services.each do |service|
|
95
|
+
if !services.include?(service.id)
|
96
|
+
members << service.members
|
97
|
+
service.remove_user(user.jid)
|
98
|
+
service.save
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# add new services
|
103
|
+
add = services.select {|id| !current.include?(id) }
|
104
|
+
Service.all.keys(add).each do |service|
|
105
|
+
members << service.members
|
106
|
+
service.add_user(user.jid)
|
107
|
+
service.save
|
108
|
+
end
|
109
|
+
|
110
|
+
System.notify_members(stream, node.from, members.flatten.uniq)
|
111
|
+
end
|
112
|
+
|
113
|
+
# Delete the user as defined by the id of the query stanza. Users may
|
114
|
+
# not delete themselves.
|
115
|
+
def delete
|
116
|
+
jid = Vines::JID.new(node.elements.first['id']).bare
|
117
|
+
raise 'jid required' if jid.empty?
|
118
|
+
user = User.find("user:#{jid}")
|
119
|
+
forbidden! unless authorized?(user) &&
|
120
|
+
node.from.stripped.to_s != jid.to_s
|
121
|
+
user.destroy
|
122
|
+
send_result
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Blather
|
4
|
+
class FileTransfer
|
5
|
+
class Ibb
|
6
|
+
# Override the accept method to write directly to the file system rather
|
7
|
+
# than using EM#attach. This is a workaround for this bug:
|
8
|
+
# https://github.com/eventmachine/eventmachine/issues/200
|
9
|
+
def accept(handler, *params)
|
10
|
+
klass = Class.new.send(:include, handler)
|
11
|
+
handler = klass.new(*params)
|
12
|
+
handler.post_init
|
13
|
+
|
14
|
+
@stream.register_handler :ibb_data, :from => @iq.from, :sid => @iq.sid do |iq|
|
15
|
+
if iq.data['seq'] == @seq.to_s
|
16
|
+
begin
|
17
|
+
handler.receive_data(Base64.decode64(iq.data.content))
|
18
|
+
@stream.write iq.reply
|
19
|
+
@seq += 1
|
20
|
+
@seq = 0 if @seq > 65535
|
21
|
+
rescue Exception => e
|
22
|
+
handler.unbind
|
23
|
+
@stream.write StanzaError.new(iq, 'not-acceptable', :cancel).to_node
|
24
|
+
end
|
25
|
+
else
|
26
|
+
handler.unbind
|
27
|
+
@stream.write StanzaError.new(iq, 'unexpected-request', :wait).to_node
|
28
|
+
end
|
29
|
+
true
|
30
|
+
end
|
31
|
+
|
32
|
+
@stream.register_handler :ibb_close, :from => @iq.from, :sid => @iq.sid do |iq|
|
33
|
+
@stream.write iq.reply
|
34
|
+
@stream.clear_handlers :ibb_data, :from => @iq.from, :sid => @iq.sid
|
35
|
+
@stream.clear_handlers :ibb_close, :from => @iq.from, :sid => @iq.sid
|
36
|
+
handler.unbind
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
@stream.clear_handlers :ibb_open, :from => @iq.from
|
41
|
+
@stream.clear_handlers :ibb_open, :from => @iq.from, :sid => @iq.sid
|
42
|
+
@stream.write @iq.reply
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module CouchRest
|
4
|
+
module RestAPI
|
5
|
+
|
6
|
+
# Wrap a blocking IO method in a new method that pushes the original method
|
7
|
+
# onto EventMachine's thread pool using EM#defer. The calling Fiber is paused
|
8
|
+
# while the thread pool does its work, then resumed when the thread finishes.
|
9
|
+
def self.defer(method)
|
10
|
+
old = "_deferred_#{method}"
|
11
|
+
alias_method old, method
|
12
|
+
define_method method do |*args|
|
13
|
+
fiber = Fiber.current
|
14
|
+
op = proc do
|
15
|
+
begin
|
16
|
+
method(old).call(*args)
|
17
|
+
rescue
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
end
|
21
|
+
cb = proc {|result| fiber.resume(result) }
|
22
|
+
EM.defer(op, cb)
|
23
|
+
Fiber.yield
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# All CouchRest::RestAPI methods ultimately call execute to connect to
|
28
|
+
# CouchDB, using blocking IO. Push those blocking calls onto the thread
|
29
|
+
# pool so we don't block the reactor thread, but still get to use all
|
30
|
+
# of the goodness of CouchRest::Model.
|
31
|
+
defer :execute
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,195 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
# Save system ohai documents in a sqlite database for fast searching. This
|
6
|
+
# index powers the live search feature of the service builder user interface.
|
7
|
+
# The indexing happens on a separate thread for two reasons:
|
8
|
+
# - writes to sqlite block the EventMachine thread
|
9
|
+
# - writes to sqlite lock the database file, so we need one writer process
|
10
|
+
class Indexer
|
11
|
+
include Vines::Log
|
12
|
+
|
13
|
+
@@indexers = {}
|
14
|
+
|
15
|
+
# Return the Indexer instance managing this file, creating a new Indexer
|
16
|
+
# instance if needed.
|
17
|
+
def self.[](file)
|
18
|
+
file = File.expand_path(file)
|
19
|
+
@@indexers[file] ||= self.new(file)
|
20
|
+
end
|
21
|
+
|
22
|
+
# There must be only one Indexer managing a sqlite file at one time.
|
23
|
+
# Use Indexer[file] to create or retrieve the Indexer for a given file
|
24
|
+
# rather than calling the constructor directly.
|
25
|
+
def initialize(file)
|
26
|
+
@db = database(file)
|
27
|
+
@tasks = PriorityQueue.new do |a, b|
|
28
|
+
a[:priority] <=> b[:priority]
|
29
|
+
end
|
30
|
+
process_tasks
|
31
|
+
end
|
32
|
+
|
33
|
+
# Queue the document for indexing at some point in the future. Because
|
34
|
+
# adding documents to the index is less time-sensitive than querying the
|
35
|
+
# index, these tasks may be delayed by query tasks.
|
36
|
+
def <<(doc)
|
37
|
+
@tasks.push({
|
38
|
+
priority: -Time.now.to_f,
|
39
|
+
type: :index,
|
40
|
+
doc: doc
|
41
|
+
})
|
42
|
+
end
|
43
|
+
|
44
|
+
# Run the SQL query with optional replacement parameters (e.g. ?) and yield
|
45
|
+
# the results array to the callback block. Queries are prioritized ahead of
|
46
|
+
# document indexing tasks so they will return quickly even when many documents
|
47
|
+
# are waiting to be indexed.
|
48
|
+
def find(query, *args, &callback)
|
49
|
+
@tasks.push({
|
50
|
+
priority: 0,
|
51
|
+
type: :query,
|
52
|
+
query: query,
|
53
|
+
args: args.flatten,
|
54
|
+
callback: callback
|
55
|
+
})
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
# Run the index processing loop, indexing one document at a time so that
|
61
|
+
# writes to sqlite are single threaded. Each task is performed in the
|
62
|
+
# EM thread pool so the reactor thread isn't blocked.
|
63
|
+
def process_tasks
|
64
|
+
@tasks.pop do |task|
|
65
|
+
callback = task[:callback]
|
66
|
+
op = proc do
|
67
|
+
case task[:type]
|
68
|
+
when :index then index(task[:doc])
|
69
|
+
when :query then query(task[:query], task[:args])
|
70
|
+
end
|
71
|
+
end
|
72
|
+
cb = proc do |results|
|
73
|
+
callback.call(results) if callback rescue nil
|
74
|
+
process_tasks
|
75
|
+
end
|
76
|
+
EM.defer(op, cb)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def query(sql, args)
|
81
|
+
@db.prepare(sql) do |stmt|
|
82
|
+
stmt.execute!(*args)
|
83
|
+
end
|
84
|
+
rescue Exception => e
|
85
|
+
log.error("Error searching index: #{e.message}")
|
86
|
+
[]
|
87
|
+
end
|
88
|
+
|
89
|
+
def index(doc)
|
90
|
+
@db.transaction do
|
91
|
+
flat = flatten(doc)
|
92
|
+
system_id = find_or_create_system(doc)
|
93
|
+
attrs = find_attributes(system_id)
|
94
|
+
delete_attributes(flat, attrs, system_id)
|
95
|
+
update_attributes(flat, attrs, system_id)
|
96
|
+
insert_attributes(flat, attrs, system_id)
|
97
|
+
end
|
98
|
+
rescue Exception => e
|
99
|
+
log.error("Error indexing document: #{e.message}")
|
100
|
+
end
|
101
|
+
|
102
|
+
def find_or_create_system(doc)
|
103
|
+
@db.prepare("select id from systems where name=?") do |stmt|
|
104
|
+
row = stmt.execute!(name(doc)).first
|
105
|
+
row ? row[0] : insert_system(doc)
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def insert_system(doc)
|
110
|
+
os = doc['kernel']['os'].downcase.sub('gnu/', '')
|
111
|
+
@db.prepare("insert into systems (name, os) values (?, ?)") do |stmt|
|
112
|
+
stmt.execute!(name(doc), os)
|
113
|
+
@db.last_insert_row_id
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def find_attributes(system_id)
|
118
|
+
sql = "select key, value from attributes where system_id=?"
|
119
|
+
@db.prepare(sql) do |stmt|
|
120
|
+
{}.tap do |attrs|
|
121
|
+
stmt.execute!(system_id) do |row|
|
122
|
+
attrs[row[0]] = row[1]
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def delete_attributes(flat, attrs, system_id)
|
129
|
+
deletes = attrs.keys.select {|k| !flat.key?(k) }
|
130
|
+
return if deletes.empty?
|
131
|
+
deletes.each_slice(999) do |slice|
|
132
|
+
params = slice.map{'?'}.join(',')
|
133
|
+
sql = "delete from attributes where system_id=? and key in (%s)" % params
|
134
|
+
@db.prepare(sql) do |stmt|
|
135
|
+
stmt.execute!(system_id, *slice)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
def update_attributes(flat, attrs, system_id)
|
141
|
+
updates = flat.select {|k, v| attrs.key?(k) && attrs[k].to_s != v.to_s }
|
142
|
+
return if updates.empty?
|
143
|
+
sql = "update attributes set value=? where system_id=? and key=?"
|
144
|
+
@db.prepare(sql) do |stmt|
|
145
|
+
updates.each do |k, v|
|
146
|
+
stmt.execute!(v, system_id, k)
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def insert_attributes(flat, attrs, system_id)
|
152
|
+
inserts = flat.select {|k, v| !attrs.key?(k) }
|
153
|
+
return if inserts.empty?
|
154
|
+
sql = "insert into attributes (system_id, key, value) values (?, ?, ?)"
|
155
|
+
@db.prepare(sql) do |stmt|
|
156
|
+
inserts.each do |k, v|
|
157
|
+
stmt.execute!(system_id, k, v)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def database(file)
|
163
|
+
SQLite3::Database.new(file).tap do |db|
|
164
|
+
db.synchronous = 'off'
|
165
|
+
db.execute("create table if not exists systems(id integer primary key, name text not null, os text)")
|
166
|
+
db.execute("create index if not exists systems_ix01 on systems (name)")
|
167
|
+
db.execute("create table if not exists attributes (system_id integer not null, key text not null, value text)")
|
168
|
+
db.execute("create index if not exists attributes_ix01 on attributes (system_id, key)")
|
169
|
+
db.execute("create index if not exists attributes_ix02 on attributes (key, value)")
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def name(doc)
|
174
|
+
doc['fqdn'].downcase
|
175
|
+
end
|
176
|
+
|
177
|
+
# Recursively expand a nested Hash into a flat key namespace. For example:
|
178
|
+
# flatten({one: {two: {three: 3}}}) #=> {"one.two.three"=>3}
|
179
|
+
def flatten(doc, output={}, stack=[])
|
180
|
+
case doc
|
181
|
+
when Hash
|
182
|
+
doc.each do |k,v|
|
183
|
+
stack.push(k)
|
184
|
+
flatten(v, output, stack)
|
185
|
+
stack.pop
|
186
|
+
end
|
187
|
+
else
|
188
|
+
val = doc.is_a?(Array) ? doc.join(',') : doc
|
189
|
+
output[stack.join('.')] = val
|
190
|
+
end
|
191
|
+
output
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# encoding: UTF-8
|
2
|
+
|
3
|
+
module Vines
|
4
|
+
module Services
|
5
|
+
|
6
|
+
# Behaves just like EM::Queue with the exception that items added to the queue
|
7
|
+
# are sorted and popped from the queue by their priority. The optional comparator
|
8
|
+
# block passed in the constructor determines element priority, with items sorted
|
9
|
+
# highest to lowest. If no block is provided, the elements' natural ordering,
|
10
|
+
# via <=>, is used.
|
11
|
+
#
|
12
|
+
# @example
|
13
|
+
#
|
14
|
+
# q = PriorityQueue.new do |a, b|
|
15
|
+
# a[:priority] <=> b[:priority]
|
16
|
+
# end
|
17
|
+
# q.push({:priority => 1, :msg => 'one'})
|
18
|
+
# q.push({:priority => 2, :msg => 'two'})
|
19
|
+
# q.push({:priority => 3, :msg => 'three'})
|
20
|
+
# 3.times do
|
21
|
+
# q.pop {|item| puts item[:msg] }
|
22
|
+
# end
|
23
|
+
# #=> "three"
|
24
|
+
# # "two"
|
25
|
+
# # "one"
|
26
|
+
#
|
27
|
+
class PriorityQueue < EM::Queue
|
28
|
+
def initialize(&comparator)
|
29
|
+
super
|
30
|
+
@items = Heap.new(&comparator)
|
31
|
+
end
|
32
|
+
|
33
|
+
# A binary max heap implementation for efficient storage of queue items. This
|
34
|
+
# class implements the Array methods called by EM::Queue so that it may
|
35
|
+
# replace the +@items+ instance variable. Namely, +push+ +shift+, +size+, and
|
36
|
+
# +empty?+ are implemented.
|
37
|
+
class Heap
|
38
|
+
def initialize(*items, &comp)
|
39
|
+
@heap = []
|
40
|
+
@comp = comp || proc {|a, b| a <=> b }
|
41
|
+
push(*items)
|
42
|
+
end
|
43
|
+
|
44
|
+
def push(*items)
|
45
|
+
items.flatten.each do |item|
|
46
|
+
@heap << item
|
47
|
+
move_up(@heap.size - 1)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
alias :<< :push
|
51
|
+
|
52
|
+
def pop
|
53
|
+
return if @heap.empty?
|
54
|
+
root = @heap[0]
|
55
|
+
@heap[0] = @heap[-1]
|
56
|
+
@heap.pop
|
57
|
+
move_down(0)
|
58
|
+
root
|
59
|
+
end
|
60
|
+
alias :shift :pop
|
61
|
+
|
62
|
+
def size
|
63
|
+
@heap.size
|
64
|
+
end
|
65
|
+
|
66
|
+
def empty?
|
67
|
+
@heap.empty?
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def move_down(k)
|
73
|
+
left = 2 * k + 1
|
74
|
+
right = 2 * k + 2
|
75
|
+
return if left > (@heap.size - 1)
|
76
|
+
larger = (right < @heap.size && @comp[@heap[right], @heap[left]] > 0) ? right : left
|
77
|
+
if @comp[@heap[k], @heap[larger]] < 0
|
78
|
+
@heap[k], @heap[larger] = @heap[larger], @heap[k]
|
79
|
+
move_down(larger)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def move_up(k)
|
84
|
+
return if k == 0
|
85
|
+
parent = (k - 1) / 2
|
86
|
+
if @comp[@heap[k], @heap[parent]] > 0
|
87
|
+
@heap[k], @heap[parent] = @heap[parent], @heap[k]
|
88
|
+
move_up(parent)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|