diaspora-vines 0.1.2

data/conf/config.rb

@@ -0,0 +1,42 @@
+Vines::Config.configure do
+  # Set the logging level to debug, info, warn, error, or fatal. The debug
+  # level logs all XML sent and received by the server.
+  log :info
+
+  # Set the directory in which to look for virtual hosts' TLS certificates.
+  # This is optional and defaults to the conf/certs directory created during
+  # `vines init`.
+  #certs 'config/vines'
+
+  # Setup a pepper to generate the encrypted password.
+  pepper "065eb8798b181ff0ea2c5c16aee0ff8b70e04e2ee6bd6e08b49da46924223e39127d5335e466207d42bf2a045c12be5f90e92012a4f05f7fc6d9f3c875f4c95b"
+
+  host 'diaspora' do
+    storage 'sql'
+  end
+
+  # Configure the client-to-server port. The max_resources_per_account attribute
+  # limits how many concurrent connections one user can have to the server.
+  client '0.0.0.0', 5222 do
+    max_stanza_size 65536
+    max_resources_per_account 5
+  end
+
+  # Configure the server-to-server port. The max_stanza_size attribute should be
+  # much larger than the setting for client-to-server.
+  server '0.0.0.0', 5269 do
+    max_stanza_size 131072
+    hosts []
+  end
+
+  # Configure the built-in HTTP server that serves static files and responds to
+  # XEP-0124 BOSH requests. This allows HTTP clients to connect to
+  # the XMPP server.
+  http '0.0.0.0', 5280 do
+    bind '/xmpp'
+    max_stanza_size 65536
+    max_resources_per_account 5
+    root 'public'
+    vroute ''
+  end
+end

data/lib/vines/cli.rb

@@ -0,0 +1,132 @@
+module Vines
+  # The command line application that's invoked by the `vines` binary included
+  # in the gem. Parses the command line arguments to create a new server
+  # directory, and starts and stops the server.
+  class CLI
+    COMMANDS = %w[start stop restart init bcrypt cert ldap schema]
+
+    def self.start
+      self.new.start
+    end
+
+    # Run the command line application to parse arguments and run sub-commands.
+    # Exits the process with a non-zero return code to indicate failure.
+    #
+    # Returns nothing.
+    def start
+      register_storage
+      opts = parse(ARGV)
+      check_config(opts)
+      command = Command.const_get(opts[:command].capitalize).new
+      begin
+        command.run(opts)
+      rescue SystemExit
+        # do nothing
+      rescue Exception => e
+        puts e.message
+        exit(1)
+      end
+    end
+
+    private
+
+    # Try to load various storage backends provided by vines-* gems and register
+    # them with the storage system for the config file to use.
+    #
+    # Returns nothing.
+    def register_storage
+      %w[couchdb mongodb redis sql].each do |backend|
+        begin
+          require 'vines/storage/%s' % backend
+        rescue LoadError
+          # do nothing
+        end
+      end
+    end
+
+    # Parse the command line arguments and run the matching sub-command
+    # (e.g. init, start, stop, etc).
+    #
+    # args - The ARGV array provided by the command line.
+    #
+    # Returns nothing.
+    def parse(args)
+      options = {}
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: vines [options] #{COMMANDS.join('|')}"
+
+        opts.separator ""
+        opts.separator "Daemon options:"
+
+        opts.on('-d', '--daemonize', 'Run daemonized in the background') do |daemonize|
+          options[:daemonize] = daemonize
+        end
+
+        options[:log] = 'log/vines.log'
+        opts.on('-l', '--log FILE', 'File to redirect output (default: log/vines.log)') do |log|
+          options[:log] = log
+        end
+
+        options[:pid] = 'pid/vines.pid'
+        opts.on('-P', '--pid FILE', 'File to store PID (default: pid/vines.pid)') do |pid|
+          options[:pid] = pid
+        end
+
+        opts.separator ""
+        opts.separator "Common options:"
+
+        opts.on('-h', '--help', 'Show this message') do |help|
+          options[:help] = help
+        end
+
+        opts.on('-v', '--version', 'Show version') do |version|
+          options[:version] = version
+        end
+      end
+
+      begin
+        parser.parse!(args)
+      rescue
+        puts parser
+        exit(1)
+      end
+
+      if options[:version]
+        puts Vines::VERSION
+        exit
+      end
+
+      if options[:help]
+        puts parser
+        exit
+      end
+
+      command = args.shift
+      unless COMMANDS.include?(command)
+        puts parser
+        exit(1)
+      end
+
+      options.tap do |opts|
+        opts[:args]    = args
+        opts[:command] = command
+        opts[:config]  = File.expand_path("#{Dir.pwd}/config/vines.rb") || File.expand_path("conf/config.rb")
+        opts[:pid]     = File.expand_path(opts[:pid])
+        opts[:log]     = File.expand_path(opts[:log])
+      end
+    end
+
+    # Many commands must be run in the context of a vines server directory
+    # created with `vines init`. If the command can't find the server's config
+    # file, print an error message and exit.
+    #
+    # Returns nothing.
+    def check_config(opts)
+      return if %w[bcrypt init].include?(opts[:command])
+      unless File.exists?(opts[:config])
+        puts "No config file found at #{opts[:config]}"
+        exit(1)
+      end
+    end
+  end
+end

data/lib/vines/cluster/connection.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Create and cache a redis database connection.
+    class Connection
+      attr_accessor :host, :port, :database, :password
+
+      def initialize
+        @redis, @host, @port, @database, @password = nil, nil, nil, nil, nil
+      end
+
+      # Return a shared redis connection.
+      def connect
+        @redis ||= create
+      end
+
+      # Return a new redis connection.
+      def create
+        conn = EM::Hiredis::Client.new(@host, @port, @password, @database)
+        conn.connect
+        conn
+      end
+    end
+  end
+end

data/lib/vines/cluster/publisher.rb

@@ -0,0 +1,55 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Broadcast messages to other cluster nodes via redis pubsub channels. All
+    # members subscribe to a channel for heartbeats, online, and offline
+    # messages from other nodes. This allows new nodes to be added to the
+    # cluster dynamically, without configuring all other nodes.
+    class Publisher
+      include Vines::Log
+
+      ALL, STANZA, USER = %w[cluster:nodes:all stanza user].map {|s| s.freeze }
+
+      def initialize(cluster)
+        @cluster = cluster
+      end
+
+      # Publish a :heartbeat, :online, or :offline message to the nodes:all
+      # broadcast channel.
+      def broadcast(type)
+        redis.publish(ALL, {
+          from: @cluster.id,
+          type: type,
+          time: Time.now.to_i
+        }.to_json)
+      end
+
+      # Send the stanza to the node hosting the user's session. The stanza is
+      # published to the channel to which the remote node is listening for
+      # messages.
+      def route(stanza, node)
+        log.debug { "Sent cluster stanza: %s -> %s\n%s\n" % [@cluster.id, node, stanza] }
+        redis.publish("cluster:nodes:#{node}", {
+          from: @cluster.id,
+          type: STANZA,
+          stanza: stanza.to_s
+        }.to_json)
+      end
+
+      # Notify the remote node that the user's roster has changed and it should
+      # reload the user from storage.
+      def update_user(jid, node)
+        redis.publish("cluster:nodes:#{node}", {
+          from: @cluster.id,
+          type: USER,
+          jid: jid.to_s
+        }.to_json)
+      end
+
+      def redis
+        @cluster.connection
+      end
+    end
+  end
+end

data/lib/vines/cluster/pubsub.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Manages the pubsub topic list and subscribers stored in redis. When a
+    # message is published to a topic, the receiving cluster node broadcasts
+    # the message to all subscribers at all other cluster nodes.
+    class PubSub
+      def initialize(cluster)
+        @cluster = cluster
+      end
+
+      # Create a pubsub topic (a.k.a. node), in the given domain, to which
+      # messages may be published. The domain argument will be one of the
+      # configured pubsub subdomains in conf/config.rb (e.g. games.wonderland.lit,
+      # topics.wonderland.lit, etc).
+      def add_node(domain, node)
+        redis.sadd("pubsub:#{domain}:nodes", node)
+      end
+
+      # Remove a pubsub topic so messages may no longer be broadcast to it.
+      def delete_node(domain, node)
+        redis.smembers("pubsub:#{domain}:subscribers_#{node}") do |subscribers|
+          redis.multi
+          subscribers.each do |jid|
+            redis.srem("pubsub:#{domain}:subscriptions_#{jid}", node)
+          end
+          redis.del("pubsub:#{domain}:subscribers_#{node}")
+          redis.srem("pubsub:#{domain}:nodes", node)
+          redis.exec
+        end
+      end
+
+      # Subscribe the JID to the pubsub topic so it will receive any messages
+      # published to it.
+      def subscribe(domain, node, jid)
+        jid = JID.new(jid)
+        redis.multi
+        redis.sadd("pubsub:#{domain}:subscribers_#{node}", jid.to_s)
+        redis.sadd("pubsub:#{domain}:subscriptions_#{jid}", node)
+        redis.exec
+      end
+
+      # Unsubscribe the JID from the pubsub topic, deregistering its interest
+      # in receiving any messages published to it.
+      def unsubscribe(domain, node, jid)
+        jid = JID.new(jid)
+        redis.multi
+        redis.srem("pubsub:#{domain}:subscribers_#{node}", jid.to_s)
+        redis.srem("pubsub:#{domain}:subscriptions_#{jid}", node)
+        redis.exec
+        redis.scard("pubsub:#{domain}:subscribers_#{node}") do |count|
+          delete_node(domain, node) if count == 0
+        end
+      end
+
+      # Unsubscribe the JID from all pubsub topics. This is useful when the
+      # JID's session ends by logout or disconnect.
+      def unsubscribe_all(domain, jid)
+        jid = JID.new(jid)
+        redis.smembers("pubsub:#{domain}:subscriptions_#{jid}") do |nodes|
+          nodes.each do |node|
+            unsubscribe(domain, node, jid)
+          end
+        end
+      end
+
+      # Return true if the pubsub topic exists and messages may be published to it.
+      def node?(domain, node)
+        @cluster.query(:sismember, "pubsub:#{domain}:nodes", node) == 1
+      end
+
+      # Return true if the JID is a registered subscriber to the pubsub topic and
+      # messages published to it should be routed to the JID.
+      def subscribed?(domain, node, jid)
+        jid = JID.new(jid)
+        @cluster.query(:sismember, "pubsub:#{domain}:subscribers_#{node}", jid.to_s) == 1
+      end
+
+      # Return a list of JIDs subscribed to the pubsub topic.
+      def subscribers(domain, node)
+        @cluster.query(:smembers, "pubsub:#{domain}:subscribers_#{node}")
+      end
+
+      private
+
+      def redis
+        @cluster.connection
+      end
+    end
+  end
+end

data/lib/vines/cluster/sessions.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Manages the cluster node list and user session routing table stored in
+    # redis. All cluster nodes share this in-memory database to quickly discover
+    # the node hosting a particular user session. Once a session is located,
+    # stanzas can be routed to that node via the +Publisher+.
+    class Sessions
+      include Vines::Log
+
+      NODES = 'cluster:nodes'.freeze
+
+      def initialize(cluster)
+        @cluster, @nodes = cluster, {}
+      end
+
+      # Return the sessions for these JIDs. If a bare JID is used, all sessions
+      # for that user will be returned. If a full JID is used, the session for
+      # that single connected stream is returned.
+      def find(*jids)
+        jids.flatten.map do |jid|
+          jid = JID.new(jid)
+          jid.bare? ? user_sessions(jid) : user_session(jid)
+        end.compact.flatten
+      end
+
+      # Persist the user's session to the shared redis cache so that other cluster
+      # nodes can locate the node hosting this user's connection and route messages
+      # to them.
+      def save(jid, attrs)
+        jid = JID.new(jid)
+        session = {node: @cluster.id}.merge(attrs)
+        redis.multi
+        redis.hset("sessions:#{jid.bare}", jid.resource, session.to_json)
+        redis.sadd("cluster:nodes:#{@cluster.id}", jid.to_s)
+        redis.exec
+      end
+
+      # Remove this user from the cluster routing table so that no further stanzas
+      # may be routed to them. This must be called when the user's session is
+      # terminated, either by logout or stream disconnect.
+      def delete(jid)
+        jid = JID.new(jid)
+        redis.hget("sessions:#{jid.bare}", jid.resource) do |response|
+          if doc = JSON.parse(response) rescue nil
+            redis.multi
+            redis.hdel("sessions:#{jid.bare}", jid.resource)
+            redis.srem("cluster:nodes:#{doc['node']}", jid.to_s)
+            redis.exec
+          end
+        end
+      end
+
+      # Remove all user sessions from the routing table associated with the
+      # given node ID. Cluster nodes call this themselves during normal shutdown.
+      # However, if a node dies without being properly shutdown, the other nodes
+      # will cleanup its sessions when they detect the node is offline.
+      def delete_all(node)
+        @nodes.delete(node)
+        redis.smembers("cluster:nodes:#{node}") do |jids|
+          redis.multi
+          redis.del("cluster:nodes:#{node}")
+          redis.hdel(NODES, node)
+          jids.each do |jid|
+            jid = JID.new(jid)
+            redis.hdel("sessions:#{jid.bare}", jid.resource)
+          end
+          redis.exec
+        end
+      end
+
+      # Cluster nodes broadcast a heartbeat to other members every second. If we
+      # haven't heard from a node in five seconds, assume it's offline and cleanup
+      # its session cache for it. Nodes may die abrubtly, without a chance to clear
+      # their sessions, so other members cleanup for them.
+      def expire
+        redis.hset(NODES, @cluster.id, Time.now.to_i)
+        redis.hgetall(NODES) do |response|
+          now = Time.now
+          expired = Hash[*response].select do |node, active|
+            offset = @nodes[node] || 0
+            (now - offset) - Time.at(active.to_i) > 5
+          end.keys
+          expired.each {|node| delete_all(node) }
+        end
+      end
+
+      # Notify the session store that this node is still alive. The node
+      # broadcasts its current time, so all cluster members' clocks don't
+      # necessarily need to be in sync.
+      def poke(node, time)
+        offset = Time.now.to_i - time
+        @nodes[node] = offset
+      end
+
+      private
+
+      # Return all remote sessions for this user's bare JID.
+      def user_sessions(jid)
+        response = @cluster.query(:hgetall, "sessions:#{jid.bare}") || []
+        Hash[*response].map do |resource, json|
+          if session = JSON.parse(json) rescue nil
+            session['jid'] = JID.new(jid.node, jid.domain, resource).to_s
+          end
+          session
+        end.compact.reject {|session| session['node'] == @cluster.id }
+      end
+
+      # Return the remote session for this full JID or nil if not found.
+      def user_session(jid)
+        response = @cluster.query(:hget, "sessions:#{jid.bare}", jid.resource)
+        return unless response
+        session = JSON.parse(response) rescue nil
+        return if session.nil? || session['node'] == @cluster.id
+        session['jid'] = jid.to_s
+        session
+      end
+
+      def redis
+        @cluster.connection
+      end
+    end
+  end
+end

data/lib/vines/cluster/subscriber.rb

@@ -0,0 +1,108 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Subscribes to the redis nodes:all broadcast channel to listen for
+    # heartbeats from other cluster members. Also subscribes to a channel
+    # exclusively for this particular node, listening for stanzas routed to us
+    # from other nodes.
+    class Subscriber
+      include Vines::Log
+
+      ALL, FROM, HEARTBEAT, OFFLINE, ONLINE, STANZA, TIME, TO, TYPE, USER =
+        %w[cluster:nodes:all from heartbeat offline online stanza time to type user].map {|s| s.freeze }
+
+      def initialize(cluster)
+        @cluster = cluster
+        @channel = "cluster:nodes:#{@cluster.id}"
+        @messages = EM::Queue.new
+        process_messages
+      end
+
+      # Create a new redis connection and subscribe to the nodes:all broadcast
+      # channel as well as the channel for this cluster node. Redis connections
+      # in subscribe mode cannot be used for other key/value operations.
+      def subscribe
+        conn = @cluster.connect
+        conn.subscribe(ALL)
+        conn.subscribe(@channel)
+        conn.on(:message) do |channel, message|
+          @messages.push([channel, message])
+        end
+      end
+
+      private
+
+      # Recursively process incoming messages from the queue, guaranteeing they
+      # are processed in the order they are received.
+      def process_messages
+        @messages.pop do |channel, message|
+          Fiber.new do
+            on_message(channel, message)
+            process_messages
+          end.resume
+        end
+      end
+
+      # Process messages as they arrive on the pubsub channels to which we're
+      # subscribed.
+      def on_message(channel, message)
+        doc = JSON.parse(message)
+        case channel
+        when ALL      then to_all(doc)
+        when @channel then to_node(doc)
+        end
+      rescue => e
+        log.error("Cluster subscription message failed: #{e}")
+      end
+
+      # Process a message sent to the nodes:all broadcast channel. In the case
+      # of node heartbeats, we update the last time we heard from this node so
+      # we can cleanup its session if it goes offline.
+      def to_all(message)
+        case message[TYPE]
+        when ONLINE, HEARTBEAT
+          @cluster.poke(message[FROM], message[TIME])
+        when OFFLINE
+          @cluster.delete_sessions(message[FROM])
+        end
+      end
+
+      # Process a message published to this node's channel. Messages sent to
+      # this channel are stanzas that need to be routed to connections attached
+      # to this node.
+      def to_node(message)
+        case message[TYPE]
+        when STANZA then route_stanza(message)
+        when USER   then update_user(message)
+        end
+      end
+
+      # Send the stanza, from a remote cluster node, to locally connected
+      # streams for the destination user.
+      def route_stanza(message)
+        node = Nokogiri::XML(message[STANZA]).root rescue nil
+        return unless node
+        log.debug { "Received cluster stanza: %s -> %s\n%s\n" % [message[FROM], @cluster.id, node] }
+        if node[TO]
+          @cluster.connected_resources(node[TO]).each do |recipient|
+            recipient.write(node)
+          end
+        else
+          log.warn("Cluster stanza missing address:\n#{node}")
+        end
+      end
+
+      # Update the roster information, that's cached in locally connected
+      # streams, for this user.
+      def update_user(message)
+        jid = JID.new(message['jid']).bare
+        if user = @cluster.storage(jid.domain).find_user(jid)
+          @cluster.connected_resources(jid).each do |stream|
+            stream.user.update_from(user)
+          end
+        end
+      end
+    end
+  end
+end