vines-services 0.1.0

data/lib/vines/services/controller/subscriptions_controller.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Controller
+      # Presence subscription requests are approved if the user has privilege to
+      # access the requested service JID. All subscriptions to the component
+      # itself are approved.
+      class SubscriptionsController < BaseController
+        register :subscription, :request?
+
+        def process
+          if approved?
+            from, to = node.from.stripped, node.to.stripped
+            stream.write(node.approve!)
+            stream.write(available(to, from))
+          else
+            stream.write(node.refuse!)
+          end
+        end
+
+        private
+
+        def approved?
+          return true if to_component?
+          found = Service.find_by_jid(node.to)
+          found && found.user?(node.from.stripped)
+        end
+
+        def available(from, to)
+          Blather::Stanza::Presence::Status.new.tap do |stanza|
+            stanza.from = from
+            stanza.to = to
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/controller/systems_controller.rb

@@ -0,0 +1,45 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Controller
+      class SystemsController < BaseController
+        register :iq, "/iq[@type='get' or @type='set']/ns:query",
+          'ns' => 'http://getvines.com/protocol/systems'
+
+        private
+
+        def get
+          if name = node.elements.first['name']
+            forbidden! unless current_user.manage_systems? || from_system?(name)
+            send_doc(System.find_by_name(name))
+          else
+            forbidden! unless current_user.manage_systems?
+            send_result(rows: System.find_all)
+          end
+        end
+
+        # Agents are allowed to save system data for their machine only. No
+        # other user may change a system's description.
+        def save
+          obj = JSON.parse(node.elements.first.content)
+          fqdn = obj['fqdn'].downcase
+          forbidden! unless from_system?(fqdn)
+
+          id = "system:#{fqdn}"
+          system = System.find(id) || System.new(id: id)
+          system.ohai = obj
+          system.save
+          storage.index(system)
+          System.notify_members(stream, node.from, [{'name' => fqdn}])
+          send_result
+        end
+
+        # Return true if a System user is requesting access to its own data.
+        def from_system?(fqdn)
+          current_user.system? && node.from.node.downcase == fqdn.downcase
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/controller/transfers_controller.rb

@@ -0,0 +1,58 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Controller
+      class TransfersController < BaseController
+        register :file_transfer
+
+        def process
+          forbidden! unless current_user.manage_files?
+          return unless node.to == stream.jid
+          name, size = node.si.file['name'], node.si.file['size'].to_i
+          log.debug("Receiving file: #{name}")
+
+          path = File.expand_path(name, uploads)
+          return unless path.start_with?(uploads)
+
+          log.debug("Saving file to #{path}")
+          File.delete(path) if File.exist?(path)
+
+          save_file_doc(name, size)
+
+          transfer = Blather::FileTransfer.new(stream, node)
+          transfer.allow_s5b = false
+          transfer.accept(FileUploader, path, size, storage)
+        end
+
+        private
+
+        def save_file_doc(name, size)
+          file = Upload.find_by_name(name) || Upload.new
+          file.name = name
+          file.size = size
+          file.save
+        end
+
+        module FileUploader
+          include Blather::FileTransfer::SimpleFileReceiver
+
+          def initialize(path, size, storage)
+            super(path, size)
+            @storage = storage
+          end
+
+          def unbind
+            super
+            return unless File.exist?(@path)
+            Fiber.new do
+              @storage.store_file(@path) do
+                File.delete(@path) rescue nil
+              end
+            end.resume
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/controller/uploads_controller.rb

@@ -0,0 +1,62 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Controller
+      class UploadsController < BaseController
+        register :iq, "/iq[@type='get' or @type='set']/ns:query",
+          'ns' => 'http://getvines.com/protocol/files'
+
+        private
+
+        def get
+          forbidden! unless current_user.manage_files?
+          id, name, label = %w[id name label].map {|a| node.elements.first[a] }
+
+          if id
+            send_doc(Upload.find(id))
+          elsif name
+            send_doc(Upload.find_by_name(name))
+          elsif label
+            send_result(rows: Upload.find_by_label(label))
+          else
+            send_result(rows: Upload.find_all)
+          end
+        end
+
+        # Uploaded files may have their names and labels updated, but not their
+        # size. New files are created by uploading them to the UploadsController,
+        # not by saving them here.
+        def save
+          forbidden! unless current_user.manage_files?
+          obj = JSON.parse(node.elements.first.content)
+          file = Upload.find(obj['id'])
+
+          unless file
+            send_error('item-not-found')
+            return
+          end
+
+          file.name = obj['name']
+          file.labels = obj['labels']
+          if file.valid?
+            file.save
+            send_doc(file)
+          else
+            send_error('not-acceptable')
+          end
+        end
+
+        def delete
+          forbidden! unless current_user.manage_files?
+          if file = Upload.find(node.elements.first['id'])
+            file.destroy
+            send_result
+          else
+            send_error('item-not-found')
+          end
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/controller/users_controller.rb

@@ -0,0 +1,127 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    module Controller
+      class UsersController < BaseController
+        register :iq, "/iq[@type='get' or @type='set']/ns:query",
+          'ns' => 'http://getvines.com/protocol/users'
+
+        private
+
+        # Returns user information to the client without the password. Passwords,
+        # despite being securely hashed, must never be sent to client requests.
+        def get
+          if jid = node.elements.first['jid']
+            user = User.find_by_jid(jid)
+            forbidden! unless authorized?(user)
+            send_doc(user)
+          else
+            users = User.find_all.select {|user| authorized?(user) }
+            send_result(rows: users)
+          end
+        end
+
+        # Save the user to the database. The password needs to be bcrypted
+        # before it is stored. Empty passwords will not be saved.
+        def save
+          obj = JSON.parse(node.elements.first.content)
+          forbidden! unless authorized?(obj)
+
+          system = (obj['system'] == true)
+          jid, name, username, password1, password2 =
+            %w[jid name username password1 password2].map {|a| (obj[a] || '').strip }
+
+          raise 'jid required' if jid.empty? && username.empty?
+
+          user = if jid.empty? # new user
+            id = "user:%s" % Vines::JID.new(username, node.from.domain)
+            raise 'user already exists' if User.find(id)
+            User.new(id: id).tap do |u|
+              u.system = system
+              u.password = password1
+            end
+          else # existing user
+            User.get!("user:%s" % Vines::JID.new(jid)).tap do |u|
+              raise "record found, but is a #{u.system ? 'system' : 'user'}" unless u.system == system
+              if system
+                u.password = password1 unless password1.empty?
+              else # humans
+                u.change_password(password1, password2) unless password1.empty? || password2.empty?
+              end
+            end
+          end
+
+          unless system
+            user.name = name
+            if user.jid == current_user.jid && !current_user.manage_services
+              if obj['permissions']['systems'] == true
+                user.valid = false
+                send_error('not-acceptable')
+              end
+            end
+            user.permissions = obj['permissions']
+          end
+
+          if user.valid?
+            user.save
+            save_services(user, obj['services'] || [])
+            send_doc(user)
+          else
+            send_error('not-acceptable')
+          end
+        end
+
+        # Return true if the user is allowed to view and save the system or
+        # user object. Users with no permissions still have the right to update
+        # their own user account.
+        def authorized?(user)
+          return false unless user
+          user = user.to_result if user.respond_to?(:to_result)
+          system = user['system'] || user[:system]
+          jid = user['jid'] || user[:jid]
+          return current_user.manage_systems? if system
+          current_user.manage_users? || current_user.jid == jid
+        end
+
+        def save_services(user, services)
+          return if user.system?
+          return unless user.permissions['manage_services']
+          current = user.services.map {|s| s.id }
+          members = []
+
+          # delete missing services
+          user.services.each do |service|
+            if !services.include?(service.id)
+              members << service.members
+              service.remove_user(user.jid)
+              service.save
+            end
+          end
+
+          # add new services
+          add = services.select {|id| !current.include?(id) }
+          Service.all.keys(add).each do |service|
+            members << service.members
+            service.add_user(user.jid)
+            service.save
+          end
+
+          System.notify_members(stream, node.from, members.flatten.uniq)
+        end
+
+        # Delete the user as defined by the id of the query stanza. Users may
+        # not delete themselves.
+        def delete
+          jid = Vines::JID.new(node.elements.first['id']).bare
+          raise 'jid required' if jid.empty?
+          user = User.find("user:#{jid}")
+          forbidden! unless authorized?(user) &&
+            node.from.stripped.to_s != jid.to_s
+          user.destroy
+          send_result
+        end
+      end
+    end
+  end
+end

data/lib/vines/services/core_ext/blather.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+
+module Blather
+  class FileTransfer
+    class Ibb
+      # Override the accept method to write directly to the file system rather
+      # than using EM#attach. This is a workaround for this bug:
+      # https://github.com/eventmachine/eventmachine/issues/200
+      def accept(handler, *params)
+        klass = Class.new.send(:include, handler)
+        handler = klass.new(*params)
+        handler.post_init
+
+        @stream.register_handler :ibb_data, :from => @iq.from, :sid => @iq.sid do |iq|
+          if iq.data['seq'] == @seq.to_s
+            begin
+              handler.receive_data(Base64.decode64(iq.data.content))
+              @stream.write iq.reply
+              @seq += 1
+              @seq = 0 if @seq > 65535
+            rescue Exception => e
+              handler.unbind
+              @stream.write StanzaError.new(iq, 'not-acceptable', :cancel).to_node
+            end
+          else
+            handler.unbind
+            @stream.write StanzaError.new(iq, 'unexpected-request', :wait).to_node
+          end
+          true
+        end
+
+        @stream.register_handler :ibb_close, :from => @iq.from, :sid => @iq.sid do |iq|
+          @stream.write iq.reply
+          @stream.clear_handlers :ibb_data, :from => @iq.from, :sid => @iq.sid
+          @stream.clear_handlers :ibb_close, :from => @iq.from, :sid => @iq.sid
+          handler.unbind
+          true
+        end
+
+        @stream.clear_handlers :ibb_open, :from => @iq.from
+        @stream.clear_handlers :ibb_open, :from => @iq.from, :sid => @iq.sid
+        @stream.write @iq.reply
+      end
+    end
+  end
+end

data/lib/vines/services/core_ext/couchrest.rb

@@ -0,0 +1,33 @@
+# encoding: UTF-8
+
+module CouchRest
+  module RestAPI
+
+    # Wrap a blocking IO method in a new method that pushes the original method
+    # onto EventMachine's thread pool using EM#defer. The calling Fiber is paused
+    # while the thread pool does its work, then resumed when the thread finishes.
+    def self.defer(method)
+      old = "_deferred_#{method}"
+      alias_method old, method
+      define_method method do |*args|
+        fiber = Fiber.current
+        op = proc do
+          begin
+            method(old).call(*args)
+          rescue
+            nil
+          end
+        end
+        cb = proc {|result| fiber.resume(result) }
+        EM.defer(op, cb)
+        Fiber.yield
+      end
+    end
+
+    # All CouchRest::RestAPI methods ultimately call execute to connect to
+    # CouchDB, using blocking IO. Push those blocking calls onto the thread
+    # pool so we don't block the reactor thread, but still get to use all
+    # of the goodness of CouchRest::Model.
+    defer :execute
+  end
+end

data/lib/vines/services/indexer.rb

@@ -0,0 +1,195 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+    # Save system ohai documents in a sqlite database for fast searching. This
+    # index powers the live search feature of the service builder user interface.
+    # The indexing happens on a separate thread for two reasons:
+    #   - writes to sqlite block the EventMachine thread
+    #   - writes to sqlite lock the database file, so we need one writer process
+    class Indexer
+      include Vines::Log
+
+      @@indexers = {}
+
+      # Return the Indexer instance managing this file, creating a new Indexer
+      # instance if needed.
+      def self.[](file)
+        file = File.expand_path(file)
+        @@indexers[file] ||= self.new(file)
+      end
+
+      # There must be only one Indexer managing a sqlite file at one time.
+      # Use Indexer[file] to create or retrieve the Indexer for a given file
+      # rather than calling the constructor directly.
+      def initialize(file)
+        @db = database(file)
+        @tasks = PriorityQueue.new do |a, b|
+          a[:priority] <=> b[:priority]
+        end
+        process_tasks
+      end
+
+      # Queue the document for indexing at some point in the future. Because
+      # adding documents to the index is less time-sensitive than querying the
+      # index, these tasks may be delayed by query tasks.
+      def <<(doc)
+        @tasks.push({
+          priority: -Time.now.to_f,
+          type: :index,
+          doc: doc
+        })
+      end
+
+      # Run the SQL query with optional replacement parameters (e.g. ?) and yield
+      # the results array to the callback block. Queries are prioritized ahead of
+      # document indexing tasks so they will return quickly even when many documents
+      # are waiting to be indexed.
+      def find(query, *args, &callback)
+        @tasks.push({
+          priority: 0,
+          type: :query,
+          query: query,
+          args: args.flatten,
+          callback: callback
+        })
+      end
+
+      private
+
+      # Run the index processing loop, indexing one document at a time so that
+      # writes to sqlite are single threaded. Each task is performed in the
+      # EM thread pool so the reactor thread isn't blocked.
+      def process_tasks
+        @tasks.pop do |task|
+          callback = task[:callback]
+          op = proc do
+            case task[:type]
+            when :index then index(task[:doc])
+            when :query then query(task[:query], task[:args])
+            end
+          end
+          cb = proc do |results|
+            callback.call(results) if callback rescue nil
+            process_tasks
+          end
+          EM.defer(op, cb)
+        end
+      end
+
+      def query(sql, args)
+        @db.prepare(sql) do |stmt|
+          stmt.execute!(*args)
+        end
+      rescue Exception => e
+        log.error("Error searching index: #{e.message}")
+        []
+      end
+
+      def index(doc)
+        @db.transaction do
+          flat = flatten(doc)
+          system_id = find_or_create_system(doc)
+          attrs = find_attributes(system_id)
+          delete_attributes(flat, attrs, system_id)
+          update_attributes(flat, attrs, system_id)
+          insert_attributes(flat, attrs, system_id)
+        end
+      rescue Exception => e
+        log.error("Error indexing document: #{e.message}")
+      end
+
+      def find_or_create_system(doc)
+        @db.prepare("select id from systems where name=?") do |stmt|
+          row = stmt.execute!(name(doc)).first
+          row ? row[0] : insert_system(doc)
+        end
+      end
+
+      def insert_system(doc)
+        os = doc['kernel']['os'].downcase.sub('gnu/', '')
+        @db.prepare("insert into systems (name, os) values (?, ?)") do |stmt|
+          stmt.execute!(name(doc), os)
+          @db.last_insert_row_id
+        end
+      end
+
+      def find_attributes(system_id)
+        sql = "select key, value from attributes where system_id=?"
+        @db.prepare(sql) do |stmt|
+          {}.tap do |attrs|
+            stmt.execute!(system_id) do |row|
+              attrs[row[0]] = row[1]
+            end
+          end
+        end
+      end
+
+      def delete_attributes(flat, attrs, system_id)
+        deletes = attrs.keys.select {|k| !flat.key?(k) }
+        return if deletes.empty?
+        deletes.each_slice(999) do |slice|
+          params = slice.map{'?'}.join(',')
+          sql = "delete from attributes where system_id=? and key in (%s)" % params
+          @db.prepare(sql) do |stmt|
+            stmt.execute!(system_id, *slice)
+          end
+        end
+      end
+
+      def update_attributes(flat, attrs, system_id)
+        updates = flat.select {|k, v| attrs.key?(k) && attrs[k].to_s != v.to_s }
+        return if updates.empty?
+        sql = "update attributes set value=? where system_id=? and key=?"
+        @db.prepare(sql) do |stmt|
+          updates.each do |k, v|
+            stmt.execute!(v, system_id, k)
+          end
+        end
+      end
+
+      def insert_attributes(flat, attrs, system_id)
+        inserts = flat.select {|k, v| !attrs.key?(k) }
+        return if inserts.empty?
+        sql = "insert into attributes (system_id, key, value) values (?, ?, ?)"
+        @db.prepare(sql) do |stmt|
+          inserts.each do |k, v|
+            stmt.execute!(system_id, k, v)
+          end
+        end
+      end
+
+      def database(file)
+        SQLite3::Database.new(file).tap do |db|
+          db.synchronous = 'off'
+          db.execute("create table if not exists systems(id integer primary key, name text not null, os text)")
+          db.execute("create index if not exists systems_ix01 on systems (name)")
+          db.execute("create table if not exists attributes (system_id integer not null, key text not null, value text)")
+          db.execute("create index if not exists attributes_ix01 on attributes (system_id, key)")
+          db.execute("create index if not exists attributes_ix02 on attributes (key, value)")
+        end
+      end
+
+      def name(doc)
+        doc['fqdn'].downcase
+      end
+
+      # Recursively expand a nested Hash into a flat key namespace. For example:
+      # flatten({one: {two: {three: 3}}}) #=> {"one.two.three"=>3}
+      def flatten(doc, output={}, stack=[])
+        case doc
+        when Hash
+          doc.each do |k,v|
+            stack.push(k)
+            flatten(v, output, stack)
+            stack.pop
+          end
+        else
+          val = doc.is_a?(Array) ? doc.join(',') : doc
+          output[stack.join('.')] = val
+        end
+        output
+      end
+    end
+  end
+end

data/lib/vines/services/priority_queue.rb

@@ -0,0 +1,94 @@
+# encoding: UTF-8
+
+module Vines
+  module Services
+
+    # Behaves just like EM::Queue with the exception that items added to the queue
+    # are sorted and popped from the queue by their priority. The optional comparator
+    # block passed in the constructor determines element priority, with items sorted
+    # highest to lowest. If no block is provided, the elements' natural ordering,
+    # via <=>, is used.
+    #
+    # @example
+    #
+    #  q = PriorityQueue.new do |a, b|
+    #    a[:priority] <=> b[:priority]
+    #  end
+    #  q.push({:priority => 1, :msg => 'one'})
+    #  q.push({:priority => 2, :msg => 'two'})
+    #  q.push({:priority => 3, :msg => 'three'})
+    #  3.times do
+    #    q.pop {|item| puts item[:msg] }
+    #  end
+    #  #=> "three"
+    #  #   "two"
+    #  #   "one"
+    #
+    class PriorityQueue < EM::Queue
+      def initialize(&comparator)
+        super
+        @items = Heap.new(&comparator)
+      end
+
+      # A binary max heap implementation for efficient storage of queue items. This
+      # class implements the Array methods called by EM::Queue so that it may
+      # replace the +@items+ instance variable. Namely, +push+ +shift+, +size+, and
+      # +empty?+ are implemented.
+      class Heap
+        def initialize(*items, &comp)
+          @heap = []
+          @comp = comp || proc {|a, b| a <=> b }
+          push(*items)
+        end
+
+        def push(*items)
+          items.flatten.each do |item|
+            @heap << item
+            move_up(@heap.size - 1)
+          end
+        end
+        alias :<< :push
+
+        def pop
+          return if @heap.empty?
+          root = @heap[0]
+          @heap[0] = @heap[-1]
+          @heap.pop
+          move_down(0)
+          root
+        end
+        alias :shift :pop
+
+        def size
+          @heap.size
+        end
+
+        def empty?
+          @heap.empty?
+        end
+
+        private
+
+        def move_down(k)
+          left  = 2 * k + 1
+          right = 2 * k + 2
+          return if left > (@heap.size - 1)
+          larger = (right < @heap.size && @comp[@heap[right], @heap[left]] > 0) ? right : left
+          if @comp[@heap[k], @heap[larger]] < 0
+            @heap[k], @heap[larger] = @heap[larger], @heap[k]
+            move_down(larger)
+          end
+        end
+
+        def move_up(k)
+          return if k == 0
+          parent = (k - 1) / 2
+          if @comp[@heap[k], @heap[parent]] > 0
+            @heap[k], @heap[parent] = @heap[parent], @heap[k]
+            move_up(parent)
+          end
+        end
+      end
+    end
+  end
+end