vines 0.3.2 → 0.4.0

data/README

@@ -6,10 +6,10 @@ CouchDB, Redis, the file system, or a custom storage implementation that you pro
 LDAP authentication can be used so user names and passwords aren't stored in the chat
 database. SSL encryption is mandatory on all client and server connections.
 
-The Vines XMPP server includes a web chat client. The web application is available 
+The Vines XMPP server includes a web chat client. The web application is available
 immediately after starting the chat server at http://localhost:5280/chat/.
 
-Additional documentation can be found at www.getvines.com.
+Additional documentation can be found at www.getvines.org.
 
 == Usage
 
@@ -21,13 +21,9 @@ Additional documentation can be found at www.getvines.com.
 
 == Dependencies
 
-* bcrypt-ruby >= 3.0.1
-* eventmachine >= 0.12.10
-* nokogiri >= 1.4.7
-* ruby >= 1.9.2
-
-== Ubuntu setup
-$ sudo apt-get install build-essential ruby1.9.1 ruby1.9.1-dev libxml2-dev libxslt-dev
+Vines requires Ruby 1.9.2 or better. Instructions for installing the
+needed OS packages, as well as Ruby itself, are available at
+http://www.getvines.org/ruby.
 
 == Contact
 

data/Rakefile

@@ -17,25 +17,27 @@ spec = Gem::Specification.new do |s|
   s.summary     = "Vines is an XMPP chat server that's easy to install and run."
   s.description = "Vines is an XMPP chat server that supports thousands of
 simultaneous connections by using EventMachine for asynchronous IO. User data
-is stored in a SQL database, CouchDB, Redis, the file system, or a custom storage
-implementation that you provide. LDAP authentication can be used so user names
-and passwords aren't stored in the chat database. SSL encryption is mandatory on
-all client and server connections."
+is stored in a SQL database, CouchDB, MongoDB, Redis, the file system, or a
+custom storage implementation that you provide. LDAP authentication can be used
+so user names and passwords aren't stored in the chat database. SSL encryption
+is mandatory on all client and server connections."
 
   s.authors      = ["David Graham"]
   s.email        = %w[david@negativecode.com]
-  s.homepage     = "http://www.getvines.com"
+  s.homepage     = "http://www.getvines.org"
 
   s.test_files   = FileList["test/**/*"]
   s.executables  = %w[vines]
   s.require_path = "lib"
 
-  s.add_dependency "activerecord", "~> 3.1.0"
+  s.add_dependency "activerecord", "~> 3.2.1"
   s.add_dependency "bcrypt-ruby", "~> 3.0.1"
-  s.add_dependency "em-http-request", "~> 0.3.0"
-  s.add_dependency "em-redis", "~> 0.3.0"
-  s.add_dependency "eventmachine", "~> 0.12.10"
+  s.add_dependency "em-http-request", "~> 1.0.1"
+  s.add_dependency "em-hiredis", "~> 0.1.0"
+  s.add_dependency "eventmachine", ">= 0.12.10"
   s.add_dependency "http_parser.rb", "~> 0.5.3"
+  s.add_dependency "mongo", "~> 1.5.2"
+  s.add_dependency "bson_ext", "~> 1.5.2"
   s.add_dependency "net-ldap", "~> 0.2.2"
   s.add_dependency "nokogiri", "~> 1.4.7"
 

data/conf/config.rb

@@ -19,25 +19,32 @@ Vines::Config.configure do
   # The private_storage attribute allows clients to store XML fragments
   # on the server, using the XEP-0049 Private XML Storage feature.
   #
+  # The pubsub attribute defines the XEP-0060 Publish-Subscribe services hosted
+  # at these virtual host domains. In the example below, pubsub services are
+  # available at games.wonderland.lit and scores.wonderland.lit as well as
+  # games.verona.lit and scores.verona.lit.
+  #
   # Shared storage example:
   # host 'verona.lit', 'wonderland.lit' do
   #   private_storage false
   #   cross_domain_messages false
   #   storage 'fs' do
-  #     dir 'data/users'
+  #     dir 'data'
   #   end
   #   components 'tea'  => 'secr3t',
   #              'cake' => 'passw0rd'
+  #   pubsub 'games', 'scores'
   # end
 
   host 'wonderland.lit' do
     cross_domain_messages false
     private_storage false
     storage 'fs' do
-      dir 'data/users'
+      dir 'data'
     end
     # components 'tea'  => 'secr3t',
     #            'cake' => 'passw0rd'
+    # pubsub 'games', 'scores'
   end
 
   # Hosts can use LDAP authentication that overrides the authentication
@@ -49,7 +56,7 @@ Vines::Config.configure do
   #   cross_domain_messages false
   #   private_storage false
   #   storage 'fs' do
-  #     dir 'data/users'
+  #     dir 'data'
   #   end
   #   ldap 'ldap.wonderland.lit', 636 do
   #     dn 'cn=Directory Manager'
@@ -98,12 +105,21 @@ Vines::Config.configure do
   component '0.0.0.0', 5347 do
     max_stanza_size 131072
   end
+
+  # Configure the redis connection used to form a cluster of server instances,
+  # serving the same chat domains across many different machines.
+  #cluster do
+  #  host 'redis.wonderland.lit'
+  #  port 6379
+  #  database 0
+  #  password ''
+  #end
 end
 
 # Available storage implementations:
 
 #storage 'fs' do
-#  dir 'data/users'
+#  dir 'data'
 #end
 
 #storage 'couchdb' do
@@ -115,6 +131,16 @@ end
 #  password ''
 #end
 
+#storage 'mongodb' do
+#  host 'localhost', 27017
+#  host 'localhost', 27018 # optional, connects to replica set
+#  database 'xmpp'
+#  tls true
+#  username ''
+#  password ''
+#  pool 5
+#end
+
 #storage 'redis' do
 #  host 'localhost'
 #  port 6379

data/lib/vines/cluster/connection.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Create and cache a redis database connection.
+    class Connection
+      attr_accessor :host, :port, :database, :password
+
+      def initialize
+        @redis = nil
+      end
+
+      # Return a shared redis connection.
+      def connect
+        @redis ||= create
+      end
+
+      # Return a new redis connection.
+      def create
+        conn = EM::Hiredis::Client.new(@host, @port, @password, @database)
+        conn.connect
+        conn
+      end
+    end
+  end
+end

data/lib/vines/cluster/publisher.rb

@@ -0,0 +1,55 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Broadcast messages to other cluster nodes via redis pubsub channels. All
+    # members subscribe to a channel for heartbeats, online, and offline
+    # messages from other nodes. This allows new nodes to be added to the
+    # cluster dynamically, without configuring all other nodes.
+    class Publisher
+      include Vines::Log
+
+      ALL, STANZA, USER = %w[cluster:nodes:all stanza user].map {|s| s.freeze }
+
+      def initialize(cluster)
+        @cluster = cluster
+      end
+
+      # Publish a :heartbeat, :online, or :offline message to the nodes:all
+      # broadcast channel.
+      def broadcast(type)
+        redis.publish(ALL, {
+          from: @cluster.id,
+          type: type,
+          time: Time.now.to_i
+        }.to_json)
+      end
+
+      # Send the stanza to the node hosting the user's session. The stanza is
+      # published to the channel to which the remote node is listening for
+      # messages.
+      def route(stanza, node)
+        log.debug { "Sent cluster stanza: %s -> %s\n%s\n" % [@cluster.id, node, stanza] }
+        redis.publish("cluster:nodes:#{node}", {
+          from: @cluster.id,
+          type: STANZA,
+          stanza: stanza.to_s
+        }.to_json)
+      end
+
+      # Notify the remote node that the user's roster has changed and it should
+      # reload the user from storage.
+      def update_user(jid, node)
+        redis.publish("cluster:nodes:#{node}", {
+          from: @cluster.id,
+          type: USER,
+          jid: jid.to_s
+        }.to_json)
+      end
+
+      def redis
+        @cluster.connection
+      end
+    end
+  end
+end

data/lib/vines/cluster/pubsub.rb

@@ -0,0 +1,92 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Manages the pubsub topic list and subscribers stored in redis. When a
+    # message is published to a topic, the receiving cluster node broadcasts
+    # the message to all subscribers at all other cluster nodes.
+    class PubSub
+      def initialize(cluster)
+        @cluster = cluster
+      end
+
+      # Create a pubsub topic (a.k.a. node), in the given domain, to which
+      # messages may be published. The domain argument will be one of the
+      # configured pubsub subdomains in conf/config.rb (e.g. games.wonderland.lit,
+      # topics.wonderland.lit, etc).
+      def add_node(domain, node)
+        redis.sadd("pubsub:#{domain}:nodes", node)
+      end
+
+      # Remove a pubsub topic so messages may no longer be broadcast to it.
+      def delete_node(domain, node)
+        redis.smembers("pubsub:#{domain}:subscribers_#{node}") do |subscribers|
+          redis.multi
+          subscribers.each do |jid|
+            redis.srem("pubsub:#{domain}:subscriptions_#{jid}", node)
+          end
+          redis.del("pubsub:#{domain}:subscribers_#{node}")
+          redis.srem("pubsub:#{domain}:nodes", node)
+          redis.exec
+        end
+      end
+
+      # Subscribe the JID to the pubsub topic so it will receive any messages
+      # published to it.
+      def subscribe(domain, node, jid)
+        jid = JID.new(jid)
+        redis.multi
+        redis.sadd("pubsub:#{domain}:subscribers_#{node}", jid.to_s)
+        redis.sadd("pubsub:#{domain}:subscriptions_#{jid}", node)
+        redis.exec
+      end
+
+      # Unsubscribe the JID from the pubsub topic, deregistering its interest
+      # in receiving any messages published to it.
+      def unsubscribe(domain, node, jid)
+        jid = JID.new(jid)
+        redis.multi
+        redis.srem("pubsub:#{domain}:subscribers_#{node}", jid.to_s)
+        redis.srem("pubsub:#{domain}:subscriptions_#{jid}", node)
+        redis.exec
+        redis.scard("pubsub:#{domain}:subscribers_#{node}") do |count|
+          delete_node(domain, node) if count == 0
+        end
+      end
+
+      # Unsubscribe the JID from all pubsub topics. This is useful when the
+      # JID's session ends by logout or disconnect.
+      def unsubscribe_all(domain, jid)
+        jid = JID.new(jid)
+        redis.smembers("pubsub:#{domain}:subscriptions_#{jid}") do |nodes|
+          nodes.each do |node|
+            unsubscribe(domain, node, jid)
+          end
+        end
+      end
+
+      # Return true if the pubsub topic exists and messages may be published to it.
+      def node?(domain, node)
+        @cluster.query(:sismember, "pubsub:#{domain}:nodes", node) == 1
+      end
+
+      # Return true if the JID is a registered subscriber to the pubsub topic and
+      # messages published to it should be routed to the JID.
+      def subscribed?(domain, node, jid)
+        jid = JID.new(jid)
+        @cluster.query(:sismember, "pubsub:#{domain}:subscribers_#{node}", jid.to_s) == 1
+      end
+
+      # Return a list of JIDs subscribed to the pubsub topic.
+      def subscribers(domain, node)
+        @cluster.query(:smembers, "pubsub:#{domain}:subscribers_#{node}")
+      end
+
+      private
+
+      def redis
+        @cluster.connection
+      end
+    end
+  end
+end

data/lib/vines/cluster/sessions.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Manages the cluster node list and user session routing table stored in
+    # redis. All cluster nodes share this in-memory database to quickly discover
+    # the node hosting a particular user session. Once a session is located,
+    # stanzas can be routed to that node via the +Publisher+.
+    class Sessions
+      include Vines::Log
+
+      NODES = 'cluster:nodes'.freeze
+
+      def initialize(cluster)
+        @cluster, @nodes = cluster, {}
+      end
+
+      # Return the sessions for these JIDs. If a bare JID is used, all sessions
+      # for that user will be returned. If a full JID is used, the session for
+      # that single connected stream is returned.
+      def find(*jids)
+        jids.flatten.map do |jid|
+          jid = JID.new(jid)
+          jid.bare? ? user_sessions(jid) : user_session(jid)
+        end.compact.flatten
+      end
+
+      # Persist the user's session to the shared redis cache so that other cluster
+      # nodes can locate the node hosting this user's connection and route messages
+      # to them.
+      def save(jid, attrs)
+        jid = JID.new(jid)
+        session = {node: @cluster.id}.merge(attrs)
+        redis.multi
+        redis.hset("sessions:#{jid.bare}", jid.resource, session.to_json)
+        redis.sadd("cluster:nodes:#{@cluster.id}", jid.to_s)
+        redis.exec
+      end
+
+      # Remove this user from the cluster routing table so that no further stanzas
+      # may be routed to them. This must be called when the user's session is
+      # terminated, either by logout or stream disconnect.
+      def delete(jid)
+        jid = JID.new(jid)
+        redis.hget("sessions:#{jid.bare}", jid.resource) do |response|
+          if doc = JSON.parse(response) rescue nil
+            redis.multi
+            redis.hdel("sessions:#{jid.bare}", jid.resource)
+            redis.srem("cluster:nodes:#{doc['node']}", jid.to_s)
+            redis.exec
+          end
+        end
+      end
+
+      # Remove all user sessions from the routing table associated with the
+      # given node ID. Cluster nodes call this themselves during normal shutdown.
+      # However, if a node dies without being properly shutdown, the other nodes
+      # will cleanup its sessions when they detect the node is offline.
+      def delete_all(node)
+        @nodes.delete(node)
+        redis.smembers("cluster:nodes:#{node}") do |jids|
+          redis.multi
+          redis.del("cluster:nodes:#{node}")
+          redis.hdel(NODES, node)
+          jids.each do |jid|
+            jid = JID.new(jid)
+            redis.hdel("sessions:#{jid.bare}", jid.resource)
+          end
+          redis.exec
+        end
+      end
+
+      # Cluster nodes broadcast a heartbeat to other members every second. If we
+      # haven't heard from a node in five seconds, assume it's offline and cleanup
+      # its session cache for it. Nodes may die abrubtly, without a chance to clear
+      # their sessions, so other members cleanup for them.
+      def expire
+        redis.hset(NODES, @cluster.id, Time.now.to_i)
+        redis.hgetall(NODES) do |response|
+          now = Time.now
+          expired = Hash[*response].select do |node, active|
+            offset = @nodes[node] || 0
+            (now - offset) - Time.at(active.to_i) > 5
+          end.keys
+          expired.each {|node| delete_all(node) }
+        end
+      end
+
+      # Notify the session store that this node is still alive. The node
+      # broadcasts its current time, so all cluster members' clocks don't
+      # necessarily need to be in sync.
+      def poke(node, time)
+        offset = Time.now.to_i - time
+        @nodes[node] = offset
+      end
+
+      private
+
+      # Return all remote sessions for this user's bare JID.
+      def user_sessions(jid)
+        response = @cluster.query(:hgetall, "sessions:#{jid.bare}") || []
+        Hash[*response].map do |resource, json|
+          if session = JSON.parse(json) rescue nil
+            session['jid'] = JID.new(jid.node, jid.domain, resource).to_s
+          end
+          session
+        end.compact.reject {|session| session['node'] == @cluster.id }
+      end
+
+      # Return the remote session for this full JID or nil if not found.
+      def user_session(jid)
+        response = @cluster.query(:hget, "sessions:#{jid.bare}", jid.resource)
+        return unless response
+        session = JSON.parse(response) rescue nil
+        return if session.nil? || session['node'] == @cluster.id
+        session['jid'] = jid.to_s
+        session
+      end
+
+      def redis
+        @cluster.connection
+      end
+    end
+  end
+end

data/lib/vines/cluster/subscriber.rb

@@ -0,0 +1,108 @@
+# encoding: UTF-8
+
+module Vines
+  class Cluster
+    # Subscribes to the redis nodes:all broadcast channel to listen for
+    # heartbeats from other cluster members. Also subscribes to a channel
+    # exclusively for this particular node, listening for stanzas routed to us
+    # from other nodes.
+    class Subscriber
+      include Vines::Log
+
+      ALL, FROM, HEARTBEAT, OFFLINE, ONLINE, STANZA, TIME, TO, TYPE, USER =
+        %w[cluster:nodes:all from heartbeat offline online stanza time to type user].map {|s| s.freeze }
+
+      def initialize(cluster)
+        @cluster = cluster
+        @channel = "cluster:nodes:#{@cluster.id}"
+        @messages = EM::Queue.new
+        process_messages
+      end
+
+      # Create a new redis connection and subscribe to the nodes:all broadcast
+      # channel as well as the channel for this cluster node. Redis connections
+      # in subscribe mode cannot be used for other key/value operations.
+      def subscribe
+        conn = @cluster.connect
+        conn.subscribe(ALL)
+        conn.subscribe(@channel)
+        conn.on(:message) do |channel, message|
+          @messages.push([channel, message])
+        end
+      end
+
+      private
+
+      # Recursively process incoming messages from the queue, guaranteeing they
+      # are processed in the order they are received.
+      def process_messages
+        @messages.pop do |channel, message|
+          Fiber.new do
+            on_message(channel, message)
+            process_messages
+          end.resume
+        end
+      end
+
+      # Process messages as they arrive on the pubsub channels to which we're
+      # subscribed.
+      def on_message(channel, message)
+        doc = JSON.parse(message)
+        case channel
+        when ALL      then to_all(doc)
+        when @channel then to_node(doc)
+        end
+      rescue Exception => e
+        log.error("Cluster subscription message failed: #{e}")
+      end
+
+      # Process a message sent to the nodes:all broadcast channel. In the case
+      # of node heartbeats, we update the last time we heard from this node so
+      # we can cleanup its session if it goes offline.
+      def to_all(message)
+        case message[TYPE]
+        when ONLINE, HEARTBEAT
+          @cluster.poke(message[FROM], message[TIME])
+        when OFFLINE
+          @cluster.delete_sessions(message[FROM])
+        end
+      end
+
+      # Process a message published to this node's channel. Messages sent to
+      # this channel are stanzas that need to be routed to connections attached
+      # to this node.
+      def to_node(message)
+        case message[TYPE]
+        when STANZA then route_stanza(message)
+        when USER   then update_user(message)
+        end
+      end
+
+      # Send the stanza, from a remote cluster node, to locally connected
+      # streams for the destination user.
+      def route_stanza(message)
+        node = Nokogiri::XML(message[STANZA]).root rescue nil
+        return unless node
+        log.debug { "Received cluster stanza: %s -> %s\n%s\n" % [message[FROM], @cluster.id, node] }
+        if node[TO]
+          @cluster.connected_resources(node[TO]).each do |recipient|
+            recipient.write(node)
+          end
+        else
+          log.warn("Cluster stanza missing address:\n#{node}")
+        end
+      end
+
+      # Update the roster information, that's cached in locally connected
+      # streams, for this user.
+      def update_user(message)
+        jid = JID.new(message['jid']).bare
+        if user = @cluster.storage(jid.domain).find_user(jid)
+          @cluster.connected_resources(jid).each do |stream|
+            stream.user.update_from(user)
+          end
+        end
+      end
+    end
+  end
+end