volt 0.9.3.pre1 → 0.9.3.pre2

data/lib/volt/server/message_bus/base_message_bus.rb

@@ -0,0 +1,52 @@
+# Volt supports the concept of a message bus, a bus provides a pub/sub interface
+# to any other volt instance (server, console, runner, etc..) inside a volt
+# cluster.  Volt ships with a PeerToPeer message bus out of the box, but you
+# can create or use other message bus's.
+#
+# MessageBus instances inherit from MessageBus::BaseMessageBus and provide
+# two methods 'publish' and 'subscribe'.  They should be inside of
+# Volt::MessageBus.
+#
+# publish should take a channel name and a message and deliver the message to
+# any subscried listeners.
+#
+# subscribe should take a channel name and a block.  It should yield a message
+# to the block if a message is published to the channel.
+#
+# The implementation details of the pub/sub connection are left to the
+# implemntation.  If the user needs to configure server addresses, Volt.config
+# is the prefered location, so it can be configured from config/app.rb
+#
+# MessageBus's should process their messages in their own thread. (And
+# optionally may use a thread pool.)
+#
+# You can use lib/volt/server/message_bus/message_encoder.rb for encoding and
+# encryption if needed.
+#
+# See lib/volt/server/message_bus/peer_to_peer.rb for details on volt's built-in
+# message bus implementation.
+#
+# NOTE: in the future, we plan to add support for round robbin message receiving
+# and other patterns.
+
+module Volt
+  module MessageBus
+    class BaseMessageBus
+      # MessagesBus's should take an instance of a Volt::App
+      def initialize(volt_app)
+        raise "Not implemented"
+      end
+
+      # Subscribe should return an object that you can call .remove on to stop
+      # the subscription.
+      def subscribe(channel_name, &block)
+        raise "Not implemented"
+      end
+
+      # publish should push out to all subscribed within the volt cluster.
+      def publish(channel_name, message)
+        raise "Not implemented"
+      end
+    end
+  end
+end

data/lib/volt/server/message_bus/message_encoder.rb

@@ -0,0 +1,64 @@
+# The message encoder handles reading/writing the message to/from the socket.
+# This includes encrypting and formatting.
+module Volt
+  module MessageBus
+    class MessageEncoder
+      attr_reader :encrypted
+      def initialize
+        # Message bus is encrypted by default
+        @encrypted = (Volt.config.message_bus.try(:disable_encryption) != true)
+
+        if @encrypted
+          # Setup a RbNaCl simple box for handling encryption
+          require 'base64'
+          begin
+            require 'rbnacl/libsodium'
+          rescue LoadError => e
+          # Ignore, incase they have libsodium installed locally
+          end
+
+          begin
+            require 'rbnacl'
+          rescue LoadError => e
+            Volt.logger.error('Volt requires the rbnacl gem to enable encryption on the message bus.  Add it to the gemfile (and rbnacl-sodium if you don\'t have libsodium installed locally')
+            raise e
+          end
+
+          # use the first 32 chars of the app secret for the encryption key.
+          key = Base64.decode64(Volt.config.app_secret)[0..31]
+
+          @encrypt_box = RbNaCl::SimpleBox.from_secret_key(key)
+        end
+      end
+
+      def encrypt(message)
+        if @encrypted
+          @encrypt_box.encrypt(message)
+        else
+          message
+        end
+      end
+
+      def decrypt(message)
+        if @encrypted
+          @encrypt_box.decrypt(message)
+        else
+          message
+        end
+      end
+
+      def send_message(io, message)
+        Marshal.dump(encrypt(message), io)
+      end
+
+      def receive_message(io)
+        begin
+          decrypt(Marshal.load(io))
+        rescue EOFError => e
+          # We get EOFError when the connection closes, return nil
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/volt/server/message_bus/peer_to_peer/peer_connection.rb

@@ -0,0 +1,186 @@
+# PeerConnection manages the connection to a peer, it takes a socket and
+# optionally the ip and port it connected to.  If ip and port are given, it
+# will try to reconnect until the server is marked as dead (as checked by
+# message_bus.still_alive?)
+
+require 'thread'
+require 'volt/server/message_bus/peer_to_peer/socket_with_timeout'
+require 'volt/server/message_bus/message_encoder'
+
+module Volt
+  module MessageBus
+    class PeerConnection
+      CONNECT_TIMEOUT = 2
+      # The server id for the connected server
+      attr_reader :peer_server_id, :socket
+
+      def initialize(socket, ip, port, message_bus, server=false)
+        @message_bus = message_bus
+        @ip = ip
+        @port = port
+        @server = server
+        @socket = socket
+        @server_id = message_bus.server_id
+        @message_queue = SizedQueue.new(500)
+        @reconnect_mutex = Mutex.new
+
+        # The encoder handles things like formatting and encryption
+        @message_encoder = MessageEncoder.new
+
+
+        failed = false
+        begin
+          if server
+            # Wait for announcement
+            @peer_server_id = @message_encoder.receive_message(@socket)
+            @message_encoder.send_message(@socket, @server_id)
+          else
+            # Announce
+            @message_encoder.send_message(@socket, @server_id)
+            @peer_server_id = @message_encoder.receive_message(@socket)
+          end
+        rescue IOError => e
+          failed = true
+        end
+
+        # Make sure we aren't already connected
+        @message_bus.remove_duplicate_connections
+
+        # Don't connect to self
+        if @failed || @peer_server_id == @server_id
+          # Close the connection
+          close
+          return
+        end
+
+        @listen_thread = Thread.new do
+          # Listen for messages in a new thread
+          listen
+        end
+
+        @worker_thread = Thread.new do
+          run_worker
+        end
+      end
+
+      # Close the socket, kill worker threads, and remove from
+      # message_bus's peer_connections
+      def disconnect
+        @disconnected = true
+        @message_queue.push(:QUIT)
+        begin
+          @socket.close
+        rescue => e
+          # Ignore close error, since we may not be connected
+        end
+
+        @listen_thread.kill
+        @worker_thread.kill
+
+        @message_bus.remove_peer_connection(self)
+      end
+
+      def publish(message)
+        @message_queue.push(message)
+      end
+
+      def run_worker
+        while (message = @message_queue.pop)
+          break if message == :QUIT
+
+          begin
+            @message_encoder.send_message(@socket, message)
+            # 'Error: closed stream' comes in sometimes
+          rescue Errno::ECONNREFUSED, Errno::EPIPE, Error => e
+            if reconnect!
+              retry
+            else
+              # Unable to reconnect, die
+              break
+            end
+          end
+        end
+      end
+
+
+      def listen
+        loop do
+          begin
+            while (message = @message_encoder.receive_message(@socket))
+              # puts "Message: #{message.inspect}"
+              break if @disconnected
+              @message_bus.handle_message(message)
+            end
+
+            # Got nil from socket
+          rescue Errno::ECONNRESET, Errno::EPIPE, IOError => e
+            # handle below
+          end
+
+          if !@disconnected && !@server
+            # Connection was dropped, try to reconnect
+            connected = reconnect!
+
+            # Couldn't reconnect, die
+            break unless connected
+          else
+            break
+          end
+        end
+      end
+
+
+      # Because servers can have many ips, we try the various ip's until we are
+      # able to connect to one.
+      # @param [Array] an array of ip strings
+      def self.connect_to(message_bus, ips, port)
+        ips.split(',').each do |ip|
+          begin
+            socket = SocketWithTimeout.new(ip, port, CONNECT_TIMEOUT)
+            return PeerConnection.new(socket, ip, port, message_bus)
+          rescue Errno::ECONNREFUSED, Errno::ETIMEDOUT => e
+            # Unable to connect, next
+            next
+          end
+        end
+
+        return false
+      end
+
+      private
+
+      def still_alive?
+        @message_bus.still_alive?(@peer_server_id)
+      end
+
+      def reconnect!
+        # Don't reconnect on the server instances
+        return false if @server
+        @reconnect_mutex.synchronize do
+          loop do
+            # Server is no longer reporting as alive, give up on reconnecting
+            unless still_alive?
+              # Unable to connect, let peer connection die
+              disconnect
+              return false
+            end
+
+            failed = false
+            begin
+              socket = SocketWithTimeout.new(@ip, @port, CONNECT_TIMEOUT)
+            rescue Errno::ECONNREFUSED, SocketError => e
+              # Unable to cnnect, wait 10, try again
+              sleep 10
+              failed = true
+            end
+
+            unless failed
+              # Reconnected
+              return true
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/volt/server/message_bus/peer_to_peer/peer_server.rb

@@ -0,0 +1,78 @@
+# The peer server opens a socket on a port that other instances (volt server,
+# runner, etc...)
+require 'socket'
+require 'thread'
+require 'volt/server/message_bus/peer_to_peer/peer_connection'
+
+module Volt
+  module MessageBus
+    class NoAvailablePortException < Exception ; end
+    class PeerServer
+      def initialize(message_bus)
+        @message_bus = message_bus
+
+        setup_port_ranges
+
+        ip = Volt.config.message_bus.try(:bind_ip)
+        begin
+          @server = TCPServer.new(random_port!)
+        rescue Errno::EADDRINUSE => e
+          # Keep trying ports until we find one that is not in use, or the pool
+          # runs out of ports.
+          retry
+        end
+
+        run_server
+      end
+
+      def run_server
+        @main_thread = Thread.new do
+          # Start the server
+          loop do
+            Thread.start(@server.accept) do |socket|
+              peer_connection = PeerConnection.new(socket, nil, nil,
+                @message_bus, true)
+
+              @message_bus.add_peer_connection(peer_connection)
+            end
+          end
+        end
+      end
+
+      def stop
+        @main_thread.kill
+      end
+
+      def port
+        @server.addr[1]
+      end
+
+      private
+        def setup_port_ranges
+          port_ranges = Volt.config.message_bus.try(:bind_port_ranges)
+
+          if port_ranges
+            # Expand any ranges, then sample one from the array
+            @ports_pool = port_ranges.to_a.map {|v| v.is_a?(Range) ? v.to_a : v }.flatten
+          else
+            # port 0, which tells TCPServer to select any random port.
+            @ports_pool = [0]
+          end
+        end
+
+        def random_port!
+          port = @ports_pool.sample
+
+          unless port
+            # no available ports left
+            raise NoAvailablePortException, 'no ports available in Volt.config.message_bus.bind_port_ranges'
+          end
+
+          # remove from the pool
+          @ports_pool.delete(port)
+
+          port
+        end
+    end
+  end
+end

data/lib/volt/server/message_bus/peer_to_peer/server_tracker.rb

@@ -0,0 +1,57 @@
+# The server tracker uses the database to keep a list of all active servers (or
+# console, runners, etc...).  When an server instance starts, it registers with
+# the database, then reads the list of all other active servers.
+
+require 'socket'
+
+module Volt
+  module MessageBus
+    class ServerTracker
+      UPDATE_INTERVAL = 10
+      def initialize(page, server_id, port)
+        @page = page
+        @server_id = server_id
+        @port = port
+
+        @main_thread = Thread.new do
+          # Continually update the database letting the server know the server
+          # is active.
+          loop do
+            begin
+              register
+            rescue Exception => e
+              puts "MessageBus Register Error: #{e.inspect}"
+            end
+            sleep UPDATE_INTERVAL
+          end
+        end
+      end
+
+      def stop
+        @main_thread.kill
+      end
+
+      # Register this server as active with the database
+      def register
+        instances = @page.store._active_volt_instances
+        instances.where(server_id: @server_id).fetch_first do |item|
+          ips = local_ips.join(',')
+          time = Time.now.to_i
+          if item
+            item.assign_attributes(ips: ips, time: time, port: @port)
+          else
+            instances << {server_id: @server_id, ips: ips, port: @port, time: time}
+          end
+        end
+      end
+
+      def local_ips
+        addr_infos = Socket.ip_address_list
+
+        ips = addr_infos.select do |addr|
+          addr.pfamily == Socket::PF_INET
+        end.map(&:ip_address)
+      end
+    end
+  end
+end

data/lib/volt/server/message_bus/peer_to_peer/socket_with_timeout.rb

@@ -0,0 +1,27 @@
+# Connect to a socket using the raw timeout, which responds better than the
+# builtin Timeout.
+
+require 'socket'
+
+module Volt
+  class SocketWithTimeout
+    def self.new(host, port, timeout=nil)
+      if RUBY_PLATFORM == 'java'
+        TCPSocket.new(host, port)
+      else
+        addr = Socket.getaddrinfo(host, nil)
+        sock = Socket.new(Socket.const_get(addr[0][0]), Socket::SOCK_STREAM, 0)
+
+        if timeout
+          secs = Integer(timeout)
+          usecs = Integer((timeout - secs) * 1_000_000)
+          optval = [secs, usecs].pack("l_2")
+          sock.setsockopt Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, optval
+          sock.setsockopt Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, optval
+        end
+        sock.connect(Socket.pack_sockaddr_in(port, addr[0][3]))
+        sock
+      end
+    end
+  end
+end

data/lib/volt/server/message_bus/peer_to_peer.rb

@@ -0,0 +1,198 @@
+# The message bus in volt is responsible for passing messages between each of
+# the running app servers (or console, etc...)
+# MessageBus::PeerToPeer is a simple message bus that automatically connects
+# any volt instance using the same database into a cluster.  It can then do
+# pub/sub between all instances.
+#
+# How It Works
+# ------------
+# Since the database is assumed to be connected to from each instance, we write
+# a record into the database with the ip of each active server and the current
+# time.
+#
+# When a server connects, it creates a socket connection to all previous servers
+# and announces its self.  Messages can then be sent over the socket.  Messages
+# are queued
+#
+# Limitations
+# -----------
+#
+# While PeerToPeer should scale fine, it currently uses threads instead of
+# an improved select library (epoll, kqueue, etc...) and non-blocking io.  The
+# plan is to rewrite it to use non-blocking io at some point.
+#
+# Also, to simplify the subscription model, messages are sent to all instances
+# reguardless of subscription status.  This greatly simplifies the
+# implementation and adds some guarentees in a distributed system, at the cost
+# of messages going to places they don't need to.  Since the primary use of the
+# message bus is alerting other instances that data has changed, this limitation
+# is not that big of an issues.  (Since the messages are small and usually need
+# to go to most servers)  That said, you may want to consider another
+# MessageBus for large scale deployments.
+#
+# The plan also is to improve the way messages are routed and to add other
+# message passing paradigms like round robin receiver.
+require 'thread'
+require 'securerandom'
+
+require 'volt/reactive/eventable'
+require 'volt/server/message_bus/peer_to_peer/server_tracker'
+require 'volt/server/message_bus/peer_to_peer/peer_server'
+require 'volt/server/message_bus/peer_to_peer/peer_connection'
+require 'volt/server/message_bus/base_message_bus'
+
+# TODO: Right now the message bus uses threads, we should switch it to use a
+# single thread and some form of select:
+# https://practicingruby.com/articles/event-loops-demystified
+
+module Volt
+  module MessageBus
+    class PeerToPeer < BaseMessageBus
+      # How long without an update before we mark an instance as dead (in seconds)
+      DEAD_TIME = 20
+      include Eventable
+
+      # Use subscribe instead of on provided in Eventable
+      alias_method :subscribe, :on
+
+      attr_reader :server_id, :page
+
+      def initialize(volt_app)
+        # Generate a guid
+        @server_id = SecureRandom.uuid
+        # The PeerConnection's to peers
+        @peer_connections = {}
+        # The server id's for each peer we're connected to
+        @peer_server_ids = {}
+
+        @page = volt_app.page
+
+        setup_peer_server
+        start_tracker
+
+        Thread.new do
+          sleep 1
+
+          connect_to_peers
+        end
+      end
+
+      # The peer server maintains a socket other instances can connect to.
+      def setup_peer_server
+        @peer_server = PeerServer.new(self)
+      end
+
+      # The tracker updates the socket ip's and port and a timestamp into the
+      # database every minute.  If the timestamp is more than 2 minutes old,
+      # an instance is marked as "dead" and removed.
+      def start_tracker
+        @server_tracker = ServerTracker.new(page, @server_id, @peer_server.port)
+
+        # Do the initial registration, and wait until its done before connecting
+        # to peers.
+        @server_tracker.register()
+      end
+
+      def publish(channel, message)
+        full_msg = "#{channel}|#{message}"
+        @peer_connections.keys.each do |peer|
+          begin
+            # Queue message on each peer
+            peer.publish(full_msg)
+          rescue IOError => e
+            # Connection to peer lost
+            Volt.logger.warn("Message bus connection to peer lost: #{e}")
+          end
+        end
+      end
+
+      # Return an array of peer records.
+      def peers
+        instances = @page.store._active_volt_instances
+
+        instances.where(server_id: {'$ne' => @server_id}).fetch.sync
+      end
+
+      def connect_to_peers
+        peers.each do |peer|
+          # Start connecting to all at the same time.  Since most will connect or
+          # timeout, this is the desired behaviour.
+          Thread.new do
+            peer_connection = PeerConnection.connect_to(self, peer._ips, peer._port)
+
+            if peer_connection
+              add_peer_connection(peer_connection)
+            else
+              # remove if not alive anymore.
+              still_alive?(peer._server_id)
+            end
+          end
+        end
+      end
+
+      def add_peer_connection(peer_connection)
+        @peer_connections[peer_connection] = true
+        @peer_server_ids[peer_connection.peer_server_id] = true
+      end
+
+      def remove_peer_connection(peer_connection)
+        @peer_connections.delete(peer_connection)
+        @peer_server_ids.delete(peer_connection.peer_server_id)
+      end
+
+      # Called when a message comes in
+      def handle_message(message)
+        channel_name, message = message.split('|', 2)
+        trigger!(channel_name, message)
+      end
+
+      # We only want one connection between two instances, this loops through each
+      # connection
+      def remove_duplicate_connections
+        peer_server_ids = {}
+
+        # remove any we are connected to twice
+        @peer_connections.keys.each do |peer|
+          peer_id = peer.peer_server_id
+
+          if peer_id
+            # peer is connected
+
+            if peer_server_ids[peer_id]
+              # Peer is already connected somewhere else, remove connection
+              peer.disconnect
+
+              # remove the connection
+              @peer_connections.delete(peer)
+            else
+              # Mark that we are connected
+              peer_server_ids[peer_id] = true
+            end
+          end
+        end
+      end
+
+      # Returns true if the server is still reporting as alive.
+      def still_alive?(peer_server_id)
+        # Unable to write to the socket, retry until the instance is no
+        # longer marking its self as active in the database
+        peer_table = @page.store._active_volt_instances
+        peer = peer_table.where(server_id: peer_server_id).fetch_first.sync
+        if peer
+          # Found the peer, retry if it has reported in in the last 2
+          # minutes.
+          if peer._time > (Time.now.to_i - DEAD_TIME)
+            # Peer reported in less than 2 minutes ago
+            return true
+          else
+            # Delete the entry
+            peer.destroy
+          end
+        end
+
+        false
+      end
+
+    end
+  end
+end

data/lib/volt/server/message_bus/redis.rb

@@ -0,0 +1 @@
+require 'volt/server/message_bus/base_message_bus'

data/lib/volt/server/rack/asset_files.rb

@@ -87,9 +87,9 @@ module Volt
 
       opal_js_files = []
       if Volt.source_maps?
-        opal_js_files += opal_files.environment['volt/page/page'].to_a.map { |v| '/assets/' + v.logical_path + '?body=1' }
+        opal_js_files += opal_files.environment['volt/volt/app'].to_a.map { |v| '/assets/' + v.logical_path + '?body=1' }
       else
-        opal_js_files << '/assets/volt/page/page.js'
+        opal_js_files << '/assets/volt/volt/app.js'
       end
       opal_js_files << '/components/main.js'
 

data/lib/volt/server/rack/component_paths.rb

@@ -17,7 +17,7 @@ module Volt
         # Gem folders with volt in them
         # TODO: we should probably qualify this a bit more
         app_folders += Gem.loaded_specs.values
-                       .select {|gem| gem.name =~ /volt/ }
+                       .select {|gem| gem.name =~ /^volt/ }
                        .map {|gem| "#{gem.full_gem_path}/app" }
 
         app_folders.uniq