palava_machine 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGNiZjliNTM4ZmUyY2E0MGQwNjRjMGQ3OGQ3MDliZWRjMmI0MTQ3OQ==
+  data.tar.gz: !binary |-
+    MDkzM2Q5MWZiZWU3YWIzZjM1NjcwNDVmYzc2M2QxNzM0ZmMxZWEzYQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    YTVkMmM3ZDUzN2UyMGQ0ZTg0NDg4YzBmNjY3ZjFmNmE1OTRkOWJjYjAzZjhh
+    YzE1ZDBhNzgwNGM3ZTRjYmNiZDgzMTA5M2NiZTc4YWUyNDU5ODc2OGEzYTg2
+    ZDJlNzM1ZDBlNjZlMDNmYTlkYjQ3NjZjNjYxYmJjYjc4MDY4YjA=
+  data.tar.gz: !binary |-
+    NjBlY2JlZDI5N2ZlYTg0NjFjNTZiNjAzZTY5YmEwMzM5YmYyZGEyZGM3MzQ5
+    MzZkZGRmMjQ5MGYyNGNlZjMyM2IwZGU3ODE0MjdhYzcwZDI4MmYxZTRkMjY3
+    MGEyNmEzMDE1MmFiNTJkMjNiOGY4ZTg0YjlhOTA4M2M1ZmJiMzU=

data/ChangeLog.md

@@ -0,0 +1,2 @@
+## 1.0.0
+- First Open Source Release

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+# ruby '2.0.0'
+ruby '1.9.3'
+gemspec

data/ProtocolChangeLog.md

@@ -0,0 +1,2 @@
+## 1.0.0
+- First Open Source Release

data/Rakefile

@@ -0,0 +1,163 @@
+require 'fileutils'
+require 'bundler/setup'
+require 'yaml'
+require 'resque/tasks'
+require 'resque_scheduler/tasks'
+
+ROOT_PATH = File.dirname(__FILE__) + '/../'
+
+# Start a worker with proper env vars and output redirection
+def run_worker(queue, count = 1)
+  puts "Starting #{count} worker(s) with QUEUE: #{queue}"
+  ops = {:pgroup => true, :err => [(ROOT_PATH + "log/resque.workers.error.log").to_s, "a"],
+                          :out => [(ROOT_PATH + "log/resque.workers.log").to_s, "a"]}
+  env_vars = {"QUEUE" => queue.to_s}
+  count.times {
+    ## Using Kernel.spawn and Process.detach because regular system() call would
+    ## cause the processes to quit when capistrano finishes
+    pid = spawn(env_vars, "rake resque:work", ops)
+    Process.detach(pid)
+  }
+end
+
+# Start a scheduler, requires resque_scheduler >= 2.0.0.f
+def run_scheduler
+  puts "Starting resque scheduler"
+  env_vars = {
+    "BACKGROUND" => "1",
+    "PIDFILE" => (ROOT_PATH + "/pid/resque-scheduler.pid").to_s,
+    "VERBOSE" => "1"
+  }
+  ops = {:pgroup => true, :err => [(ROOT_PATH + "log/resque.scheduler.error.log").to_s, "a"],
+                          :out => [(ROOT_PATH + "log/resque.scheduler.log").to_s, "a"]}
+  pid = spawn(env_vars, "rake resque:scheduler", ops)
+  Process.detach(pid)
+end
+
+
+namespace :deploy do
+  # desc 'whiskey_disk deploy hook'
+  # task :post_deploy => %w[
+  #   resque:restart_workers
+  #   resque:restart_scheduler
+  # ]
+end
+
+
+namespace :resque do
+  task :environment do
+    require 'resque'
+    require 'resque_scheduler'
+    require 'resque/scheduler'
+
+    Resque.redis = 'localhost:6379'
+    Resque.schedule = YAML.load_file('config/schedule.yml')
+    require_relative 'jobs'
+  end
+
+  task :setup => :environment
+
+  desc "Restart running workers"
+  task :restart_workers => :environment do
+    Rake::Task['resque:stop_workers'].invoke
+    Rake::Task['resque:start_workers'].invoke
+  end
+
+  desc "Quit running workers"
+  task :stop_workers => :environment do
+    pids = Array.new
+    Resque.workers.each do |worker|
+      pids.concat(worker.worker_pids)
+    end
+    if pids.empty?
+      puts "No workers to kill"
+    else
+      syscmd = "kill -s QUIT #{pids.join(' ')}"
+      puts "Running syscmd: #{syscmd}"
+      system(syscmd)
+    end
+  end
+
+  desc "Start workers"
+  task :start_workers => :environment do
+    run_worker("*", 2)
+    run_worker("high", 1)
+  end
+
+  desc "Restart scheduler"
+  task :restart_scheduler => :environment do
+    Rake::Task['resque:stop_scheduler'].invoke
+    Rake::Task['resque:start_scheduler'].invoke
+  end
+
+  desc "Quit scheduler"
+  task :stop_scheduler => :environment do
+    pidfile = ROOT_PATH + "pid/resque-scheduler.pid"
+    if !File.exists?(pidfile)
+      puts "Scheduler not running"
+    else
+      pid = File.read(pidfile).to_i
+      syscmd = "kill -s QUIT #{pid}"
+      puts "Running syscmd: #{syscmd}"
+      system(syscmd)
+      FileUtils.rm_f(pidfile)
+    end
+  end
+
+  desc "Start scheduler"
+  task :start_scheduler => :environment do
+    run_scheduler
+  end
+
+  desc "Reload schedule"
+  task :reload_schedule => :environment do
+    pidfile = ROOT_PATH + "pid/resque-scheduler.pid"
+
+    if !File.exists?(pidfile)
+      puts "Scheduler not running"
+    else
+      pid = File.read(pidfile).to_i
+      syscmd = "kill -s USR2 #{pid}"
+      puts "Running syscmd: #{syscmd}"
+      system(syscmd)
+    end
+  end
+end
+
+# # #
+
+def gemspec
+  name = Dir['*.gemspec'].first
+  @gemspec ||= eval(File.read(name), binding, name)
+end
+
+desc "Build the gem"
+task :gem => :gemspec do
+  sh "gem build #{gemspec.name}.gemspec"
+  FileUtils.mkdir_p 'pkg'
+  FileUtils.mv "#{gemspec.name}-#{gemspec.version}.gem", 'pkg'
+end
+
+desc "Install the gem locally"
+task :install => :gem do
+  sh %{gem install pkg/#{gemspec.name}-#{gemspec.version}.gem --no-rdoc --no-ri}
+end
+
+desc "Generate the gemspec"
+task :generate do
+  puts gemspec.to_ruby
+end
+
+desc "Validate the gemspec"
+task :gemspec do
+  gemspec.validate
+end
+
+desc 'rspec specs'
+task :spec do
+  sh %[rspec spec]
+end
+
+task :default => :spec
+task :test    => :spec
+

data/ReadMe.md

@@ -0,0 +1,56 @@
+# PalavaMachine
+
+## Description
+
+PalavaMachine is a WebRTC signaling server. Signaling describes the process of finding other peers and exchange information about how to establish a media connection.
+
+The server is implemented in [EventMachine](http://rubyeventmachine.com/) and [Redis PubSub](http://redis.io/topics/pubsub) and communication to the clients is done via WebSockets. See it in action at [palava.tv.](https://palava.tv)
+
+## What can I do with it?
+
+*This is a pre-release for interested Ruby/JS/WebRTC developers*. If you are unsure, what to use this gem for, you'll just need to wait. We'll soon put a more detailed instructions on our [blog](https://blog.palava.tv).
+
+## Installation & Usage
+
+Make sure you have redis(http://redis.io/download) installed, then clone this repository and run
+
+  $ bundle install
+
+Start the server with
+
+  $ bin/palava-machine
+
+Alternatively, download the [palava_machine gem](http://rubygems.org/gems/palava_machine) from rubygems.org:
+
+  $ gem install palava_machine
+
+And run:
+
+  $ palava-machine
+
+### Deamonized Version
+
+The PalavaMachine can be started as a daemon process for production usage:
+
+  $ palava-machine-daemon start
+
+Stop it with
+
+  $ palava-machine-daemon stop
+
+### Specs
+
+To run the test suite use
+
+  $ rspec
+
+## Credits
+
+Open Source License information following soon!
+
+(c) 2013 Jan Lelis,      jan@signaling.io
+(c) 2013 Marius Melzer,  marius@signaling.io
+(c) 2013 Stephan Thamm,  thammi@chaossource.net
+(c) 2013 Kilian Ulbrich, kilian@innovailable.eu
+
+Part of the [palava project](https://blog.palava.tv)

data/bin/palava-machine

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/palava_machine/runner'
+PalavaMachine::Runner.run(PalavaMachine::Runner.parse_cli_options)

data/bin/palava-machine-daemon

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+require 'daemons'
+
+Daemons.run("#{File.dirname(__FILE__)}/palava-machine",
+  multiple: true,
+  ARGV: ARGV + ['--', '4240', '5']
+)

data/lib/palava_machine.rb

@@ -0,0 +1,13 @@
+require_relative 'palava_machine/version' unless defined? PalavaMachine::VERSION
+
+module PalavaMachine
+  class MessageParsingError < StandardError; end
+
+  class MessageError < StandardError
+    attr_reader :ws
+
+    def initialize(ws)
+      @ws = ws
+    end
+  end
+end

data/lib/palava_machine/client_message.rb

@@ -0,0 +1,46 @@
+require_relative 'version'
+
+require 'json'
+
+module PalavaMachine
+  class ClientMessage
+    RULES = {
+      info:          [],
+      join_room:     [:room_id, :status],
+      leave_room:    [],
+      send_to_peer:  [:peer_id, :data],
+      update_status: [:status],
+    }
+
+    def initialize(message, connection_id = nil)
+      begin
+        @_data = JSON.parse(message)
+      rescue # TODO find exact json error to catch
+        raise MessageParsingError, 'invalid message'
+      end
+
+      raise MessageParsingError, 'invalid message: not a hash' unless @_data.instance_of?(Hash)
+      @connection_id = connection_id
+    end
+
+    def [](w)
+      @_data[w]
+    end
+
+    def valid?
+      RULES.keys.include?(name) or raise MessageParsingError, 'unknown event'
+    end
+
+    def name
+      @name ||= @_data['event'] && @_data['event'].to_sym or raise(MessageParsingError, 'no event given')
+    end
+
+    def connection_id
+      @connection_id or raise MessageParsingError, 'connection id used but not set'
+    end
+
+    def arguments
+      valid? && RULES[name].map{ |data_key| @_data[data_key.to_s] }
+    end
+  end
+end

data/lib/palava_machine/jobs.rb

@@ -0,0 +1,3 @@
+Dir[File.dirname(__FILE__) + '/jobs/*'].each { |filename|
+  require_relative "jobs/#{ File.basename(filename) }"
+}

data/lib/palava_machine/jobs/export_stats_job.rb

@@ -0,0 +1,59 @@
+require 'set'
+require 'redis'
+require 'mongo'
+
+class ExportStatsJob
+  class StatsExporter
+    STATS_NAMESPACE = "store:stats"
+
+    def initialize(redis_address, mongo_address)
+      @redis = Redis.new(host: 'localhost', port: 6379)
+      @mongo = Mongo::MongoClient.new#(mongo_address)
+      @times = Set.new
+    end
+
+    def import_timestamps!(ns)
+      redis_pattern = "#{STATS_NAMESPACE}:#{ns}:*"
+      offset = redis_pattern.size - 1
+      @times.merge @redis.keys(redis_pattern).map{ |key|
+        key[offset..-1].to_i
+      }
+    end
+
+    # remove timestamps which are not closed (+ grace time)
+    def prune_timestamps!
+      limit = Time.now.utc.to_i - 3660
+      @times.select!{ |time| time < limit }
+      puts "Transfering #{@times.length} timespans"
+    end
+
+    def store_in_mongo!
+      collection = @mongo.db("plv_stats").collection("rtc")
+
+      @times.each { |time|
+        collection.insert(
+          "c_at"            => time,
+          "connection_time" => get_and_delete_from_redis("connection_time", time),
+          "room_peaks"      => get_and_delete_from_redis("room_peaks",      time),
+        )
+      }
+    end
+
+    def get_and_delete_from_redis(ns, time)
+      key = "#{STATS_NAMESPACE}:#{ns}:#{time}"
+      data = @redis.hgetall(key) || {}
+      @redis.del(key)
+      data
+    end
+  end
+
+  class << self
+    def perform(redis_address = 'localhost:6379', mongo_address = 'localhost:27017')
+      se = StatsExporter.new(redis_address, mongo_address)
+      se.import_timestamps! "room_peaks"
+      se.import_timestamps! "connection_time"
+      se.prune_timestamps!
+      se.store_in_mongo!
+    end
+  end
+end

data/lib/palava_machine/manager.rb

@@ -0,0 +1,311 @@
+require_relative 'version'
+require_relative 'socket_store'
+
+require 'em-hiredis'
+require 'json'
+require 'digest/sha2'
+require 'time'
+require 'logger'
+require 'logger/colors'
+require 'forwardable'
+
+
+module PalavaMachine
+  class Manager
+    extend Forwardable
+
+
+    attr_reader :connections
+
+
+    def_delegators :@log, :debug, :info, :warn, :error, :fatal
+
+
+    PAYLOAD_NEW_PEER = lambda { |connection_id, status = nil|
+      payload = { event: 'new_peer', peer_id: connection_id }
+      payload[:status] = status if status
+      payload.to_json
+    }
+
+    PAYLOAD_PEER_LEFT = lambda { |connection_id| {
+      event: 'peer_left',
+      sender_id: connection_id,
+    }.to_json }
+
+
+    SCRIPT_JOIN_ROOM = <<-LUA
+      local members = redis.call('smembers', KEYS[1])
+      local count = 0
+      for _, peer_id in pairs(members) do
+        redis.call('publish', "ps:connection:" .. peer_id, ARGV[2])
+        count = count + 1
+      end
+      redis.call('sadd', KEYS[1], ARGV[1])
+      if count == 0 or tonumber(redis.call('get', KEYS[2])) <= count then
+        redis.call('set', KEYS[2], count + 1)
+      end
+      redis.call('set', KEYS[3], ARGV[3])
+      redis.call('set', KEYS[4], ARGV[4])
+      return members
+    LUA
+
+    SCRIPT_LEAVE_ROOM = <<-LUA
+      redis.call('hincrby', KEYS[7], (ARGV[3] - tonumber(redis.call('get', KEYS[3]))) / 60, 1) --stats
+      redis.call('srem', KEYS[1], ARGV[1])
+      redis.call('del', KEYS[3])
+      redis.call('del', KEYS[4])
+      redis.call('del', KEYS[5])
+
+      if redis.call('scard', KEYS[1]) == 0 then -- also delete room if it is empty
+        redis.call('hincrby', KEYS[6], redis.call('get', KEYS[2]), 1) --stats
+        redis.call('del', KEYS[1])
+        redis.call('del', KEYS[2])
+      else -- tell others in room
+        for _, peer_id in pairs(redis.call('smembers', KEYS[1])) do
+          redis.call('publish', "ps:connection:" .. peer_id, ARGV[2])
+        end
+      end
+    LUA
+
+
+    def initialize(options = {})
+      @redis_address = 'localhost:6379'
+      @redis_db      = options[:db] || 0
+      @connections   = SocketStore.new
+      @log           = Logger.new(STDOUT)
+      @log.level     = Logger::DEBUG
+      @log.formatter = proc{ |level, datetime, _, msg|
+        "#{datetime.strftime '%F %T'} | #{msg}\n"
+      }
+    end
+
+    def initialize_in_em
+      @redis      = EM::Hiredis.connect "redis://#{@redis_address}/#{@redis_db}"
+      @publisher  = @redis.pubsub
+      @subscriber = EM::Hiredis.connect("redis://#{@redis_address}/#{@redis_db}").pubsub # You need an extra connection for subs
+      @redis.on :failed do
+        @log.error 'Could not connect to Redis server'
+      end
+    end
+
+    def announce_connection(ws)
+      connection_id = @connections.register_connection(ws)
+      info "#{connection_id} <open>"
+
+      @subscriber.subscribe "ps:connection:#{connection_id}" do |payload|
+        # debug "SUB payload #{payload} for <#{connection_id}>"
+        ws.send_text(payload)
+      end
+    end
+
+    def return_error(connection_id, message)
+      raise MessageError.new(@connections[connection_id]), message
+    end
+
+    def unannounce_connection(ws, close_ws = false)
+      if connection_id = @connections.unregister_connection(ws)
+        info "#{connection_id} <close>"
+        leave_room(connection_id)
+        @subscriber.unsubscribe "ps:connection:#{connection_id}"
+        if close_ws && ws.state != :closed # currently not used FIXME
+          ws.close
+        end
+      end
+    end
+
+    def join_room(connection_id, room_id, status)
+      return_error connection_id, 'no room id given' if !room_id || room_id.empty?
+      return_error connection_id, 'room id too long' if room_id.size > 50
+
+      @redis.get "store:connection:room:#{connection_id}" do |res|
+        return_error connection_id, 'already joined another room' if res
+        room_id = Digest::SHA512.hexdigest(room_id)
+        info "#{connection_id} joins ##{room_id[0..10]}... #{status}"
+
+        script_join_room(connection_id, room_id, status){ |members|
+          return_error connection_id, 'room is full' unless members
+
+          update_status_without_notifying_peers(connection_id, status){
+            if members.empty?
+              send_joined_room(connection_id, [])
+            else
+              get_statuses_for_members(members) do |members_with_statuses|
+                send_joined_room(connection_id, members_with_statuses)
+              end
+            end
+          }
+        }
+      end
+    end
+
+    def script_join_room(connection_id, room_id, status, &block)
+      @redis.eval \
+        SCRIPT_JOIN_ROOM,
+        4,
+        "store:room:members:#{room_id}",
+        "store:room:peak_members:#{room_id}",
+        "store:connection:joined:#{connection_id}",
+        "store:connection:room:#{connection_id}",
+        connection_id,
+        PAYLOAD_NEW_PEER[connection_id, status],
+        Time.now.getutc.to_i,
+        room_id,
+        &block
+    end
+    private :script_join_room
+
+    def get_statuses_for_members(members)
+      member_count = members.size
+      members_with_statuses = []
+      members.each { |peer_id|
+        @redis.hgetall("store:connection:status:#{peer_id}") do |status_array|
+          members_with_statuses << { peer_id: peer_id, status: Hash[status_array.each_slice(2).to_a] }
+          yield members_with_statuses if members_with_statuses.size == member_count
+        end
+      }
+    end
+    private :get_statuses_for_members
+
+    def send_joined_room(connection_id, members_with_statuses)
+      @connections[connection_id].send_text({
+        event: 'joined_room',
+        own_id: connection_id,
+        peers: members_with_statuses,
+      }.to_json)
+    end
+    private :send_joined_room
+
+    def leave_room(connection_id)
+      @redis.get("store:connection:room:#{connection_id}") do |room_id|
+        next unless room_id # return_error connection_id, 'currently not in any room'
+
+        info "#{connection_id} leaves ##{room_id[0..10]}..."
+        script_leave_room(connection_id, room_id)
+      end
+    end
+
+    def script_leave_room(connection_id, room_id, &block)
+      now = Time.now.getutc.to_i
+      hour = now - now % (60 * 60)
+
+      @redis.eval \
+        SCRIPT_LEAVE_ROOM,
+        7,
+        "store:room:members:#{room_id}",
+        "store:room:peak_members:#{room_id}",
+        "store:connection:joined:#{connection_id}",
+        "store:connection:room:#{connection_id}",
+        "store:connection:status:#{connection_id}",
+        "store:stats:room_peaks:#{hour}",
+        "store:stats:connection_time:#{hour}",
+        connection_id,
+        PAYLOAD_PEER_LEFT[connection_id],
+        now,
+        &block
+    end
+    private :script_leave_room
+
+    def update_status(connection_id, input_status)
+      @redis.get("store:connection:room:#{connection_id}") do |room_id|
+        return_error connection_id, 'currently not in any room' unless room_id
+
+        update_status_without_notifying_peers(connection_id, input_status){
+          @redis.smembers("store:room:members:#{room_id}") do |members|
+            members.each { |peer_id|
+              @publisher.publish "ps:connection:#{peer_id}", {
+                event: 'peer_updated_status',
+                status: input_status,
+                sender_id: connection_id,
+              }.to_json
+            }
+          end
+        }
+      end
+    end
+
+    def send_to_peer(connection_id, peer_id, data)
+      unless data.instance_of? Hash
+        return_error connection_id, "cannot send raw data"
+      end
+
+      @redis.get("store:connection:room:#{connection_id}") do |room_id|
+        return_error connection_id, 'currently not in any room' unless room_id
+
+        @redis.sismember("store:room:members:#{room_id}", peer_id) do |is_member|
+          return_error connection_id, 'unknown peer' if is_member.nil? || is_member.zero?
+
+          unless %w[offer answer ice_candidate].include? data['event']
+            return_error connection_id, 'event not allowed'
+          end
+
+          @publisher.publish "ps:connection:#{peer_id}", (data || {}).merge("sender_id" => connection_id).to_json
+        end
+      end
+    end
+
+    def announce_shutdown(seconds = 0)
+      warn "Announcing shutdown in #{seconds} seconds"
+      @connections.sockets.each { |ws|
+        ws.send_text({
+          event: 'shutdown',
+          seconds: seconds,
+        }.to_json)
+      }
+    end
+
+    def shutdown!(seconds = 0)
+      sleep(seconds)
+      @connections.dup.sockets.each{ |ws| ws.close(4200) } # TODO double check this one
+    end
+
+
+    private
+
+
+    # TODO shorten
+    def update_status_without_notifying_peers(connection_id, input_status, &block)
+      if !input_status
+        block.call
+        return false
+      end
+
+      status = {}
+
+      if input_status['name']
+        if !input_status['name'] || input_status['name'] =~ /\A\s*\z/
+          return_error connection_id, 'blank name not allowed'
+        end
+
+        if input_status['name'].size > 50
+          return_error connection_id, 'name too long'
+        end
+
+        begin
+          valid_encoding = input_status['name'] =~ /\A\p{ASCII}+\z/
+        rescue Encoding::CompatibilityError
+          valid_encoding = false
+        end
+
+        if !valid_encoding
+          input_status['name'] = '*' * input_status['name'].size
+        end
+
+        status['name'] = input_status['name']
+      end
+
+      if input_status['user_agent']
+        unless %w[firefox chrome unknown].include? input_status['user_agent']
+          return_error connection_id, 'unknown user agent'
+        end
+
+        status['user_agent'] = input_status['user_agent']
+      end
+
+      unless status.empty?
+        @redis.hmset "store:connection:status:#{connection_id}", *status.to_a.flatten, &block
+        true
+      end
+    end
+
+  end
+end