seapig-server 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 867ee8657bbc0f1f9cdf2f8d56627179e10860f2
-  data.tar.gz: e9102a3be76c639d1717d45c1366271cc9f97721
+  metadata.gz: 02ea92f8574a95efe7cbb84a4273a3f333fa0104
+  data.tar.gz: d7d0b375e2b6d801387c14420eebc938f15ad2ab
 SHA512:
-  metadata.gz: eb28661eaa20bad43f3935c63b66a8ce0ba95febba2f8da5bd0742ab9d1b4e0acadd8eecdbcf6f59e31ebcefca57ed34c5558a9e74a60d88527d09bf7e82d3b8
-  data.tar.gz: 18d635f7e0ef27f966a4d858e8ea8c4696942f4c780c27635a58ad96d754564f5bfb29c66714f560e3f1adc172971143563c184605c525608573dffe95212c80
+  metadata.gz: ff15f55ebfcce98672f6a820a54c254c1f4d6b5c3055fb4fd50975bec7021291d4fde0535edd1d6b84ee154725c3d315261243a543f9cc91b4c98ce4c118ff72
+  data.tar.gz: 17650ede8b9698155b93a1699ee61b9f597d805602c77eda65af2c0428d0e7eace0803c02b41e6af4f08292e92f3588fb9883093a1cfafa5a62c49eefe226dc9

data/MIT-LICENSE

@@ -1,4 +1,4 @@
-Copyright 2015 yunta
+Copyright 2015-2017 yunta
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.rdoc

@@ -1,36 +1,31 @@
 = Seapig
 
-Nothing here yet.
+Seapig is a websocket-based master-slave json object replication/synchronisation system.
+
+No docs exist yet.
 
 To cover:
 
-* what are seapigs and why they exist? (why not just having a db table?)
+* what are seapigs and why they exist? (why not just having a stream?)
   * link to https://www.youtube.com/watch?v=_y4DbZivHCY
-* properties of seapig system
-  * data is cached in memory
-  * only diffs are sent
-  * data is generated in separate processes
-  * data gets regenerated on dependency change (immediately on db change)
+* properties of a seapig system
+  * objects are cached in memory
+  * only diffs are sent over websocket when objects change
+  * seapig handles re-synchronization of objects on link loss
+  * objects are generated in separate processes
+  * objects are regenerated on dependency change (e.g. immediately on db change)
 * describe current limits
-  * one level deps only
-  * single object can have at most one generation process running at all times
-  * cache is dropped when last client unlinks
-  * no client-side timeout detection
-  * postgres only (but is that really a problem? ;))
+  * there is no rate-limiting
+  * object is dropped from cache when last client un-listens
+  * seapig-client: no timeout detection
+  * seapig-rails: postgres only (but is that really a problem? ;))
 * graph of server/client side seapig object states
-* disclaimer on non-efficient code. it's all a draft, a test of idea.
+* disclaimer on non-efficient code. it's all a draft, a test of an idea
 * rails not needed
 * stuff will change
-* seapig = firebase for apps that need real db
+* seapig == firebase for apps that need real db
 * works nicely with mithril
 * how to use this shit
-  * dummy app is a minimal demo
-  * bundle exec rails s
-  * bundle exec ruby ../../bin/seapig-server.rb
-  * bundle exec ruby ../../bin/seapig-worker.rb ws://127.0.0.1:3001/seapig
-  * bundle exec ruby ../../bin/seapig-notifier.rb ws://127.0.0.1:3001/seapig
-  * application.js needs: require seapig/seapig
-  * application.js needs: require json-patch
-  * ActiveRecord models that are used for triggering regeneration of data need:
-    * acts_as_seapig_dependency
-    * and seapig_dependency_changed after commits
+  * write some docs
+  * link to example projects
+* link to seapig-rails, seapig-client-ruby, seapig-router, etc.

data/bin/seapig-server

@@ -112,23 +112,23 @@ module SeapigObjectStore
 
 
 	def self.consumer_unregister(pattern_or_id, client)
-		raise "Unregister without register" if not @@consumers[pattern_or_id].include?(client)
+		raise "Unregister without register" if (not @@consumers[pattern_or_id]) or (not @@consumers[pattern_or_id].include?(client))
 		@@consumers[pattern_or_id].delete(client)
 		@@consumers.delete(pattern_or_id) if @@consumers[pattern_or_id].size == 0
 		self.matching(pattern_or_id,@@producers.merge(@@objects_by_id)).each { |matching_id|
 			@@objects_by_id[matching_id].consumer_unregister(pattern_or_id, client) if @@objects_by_id[matching_id]
-			self.despawn(@@objects_by_id[matching_id]) if @@objects_by_id[matching_id] and (not @@objects_by_id[matching_id].alive?) and (not @@dependents[pattern_or_id])
+			self.despawn(@@objects_by_id[matching_id]) if @@objects_by_id[matching_id] and (not @@objects_by_id[matching_id].alive?) and (not @@dependents[matching_id])
 		}
 	end
 
 
 	def self.producer_unregister(pattern_or_id,client)
-		raise "Unregister without register" if not @@producers[pattern_or_id].include?(client)
+		raise "Unregister without register" if (not @@producers[pattern_or_id]) or (not @@producers[pattern_or_id].include?(client))
 		@@producers[pattern_or_id].delete(client)
 		@@producers.delete(pattern_or_id) if @@producers[pattern_or_id].size == 0
 		self.matching(pattern_or_id,@@consumers.merge(@@dependents)).each { |matching_id|
 			@@objects_by_id[matching_id].producer_unregister(pattern_or_id, client) if @@objects_by_id[matching_id]
-			self.despawn(@@objects_by_id[matching_id]) if @@objects_by_id[matching_id] and (not @@objects_by_id[matching_id].alive?) and (not @@dependents[pattern_or_id])
+			self.despawn(@@objects_by_id[matching_id]) if @@objects_by_id[matching_id] and (not @@objects_by_id[matching_id].alive?) and (not @@dependents[matching_id])
 		}
 	end
 
@@ -481,6 +481,11 @@ class Client
 	end
 
 
+	def inspect
+		id
+	end
+
+
 	def destroy
 		puts "Client disconnected:\n        "+@index.to_s if DEBUG
 		@@clients_by_socket.delete(@socket)
@@ -637,22 +642,6 @@ class Client
 end
 
 
-class InternalClient
-
-	def self.produce
-	end
-
-	def initialize
-		SeapigObjectStore.producer_register("SeapigServer::Objects", self)
-	end
-
-	def object_produce(object_id, object_version)
-		objects =
-		SeapigObjectStore.version_set(object_id,new_version,objects,object_version)
-	end
-
-end
-
 
 #TODO:
 # * change protocol to use "pattern" instead of "id"
@@ -734,4 +723,19 @@ EM.run {
 	}
 
 
+	close_reader, close_writer = IO.pipe
+
+	EM.watch(close_reader) { |connection|
+		connection.notify_readable = true
+		connection.define_singleton_method(:notify_readable) do
+			puts "Shutting down" if INFO or DEBUG
+			exit
+		end
+	}
+
+	Signal.trap("INT") {
+		puts "SIGINT received, scheduling exit." if DEBUG
+		close_writer.write('.')
+	}
+
 }

data/bin/seapig-server-intro

@@ -0,0 +1,757 @@
+#!/bin/env ruby
+# coding: utf-8
+require 'websocket-eventmachine-server'
+require 'narray'
+require 'oj'
+require 'jsondiff'
+require 'hana'
+require 'set'
+
+
+DEBUG = (ARGV[0] == "debug")
+INFO = (DEBUG or ARGV[0] == "info")
+HOST = (ARGV[1] or "127.0.0.1").split(":")[0]
+PORT = ((ARGV[1] or '').split(':')[1] or "3001").to_i
+
+OBJECT_CACHE_SIZE = 1
+
+$stdout.sync = true
+
+Oj.default_options = { mode: :compat }
+
+
+module WebSocket
+	module Frame
+		class Data < String
+			def getbytes(start_index, count)
+				data = self[start_index, count]
+				if @masking_key
+					payload_na = NArray.to_na(data,"byte")
+					mask_na = NArray.to_na((@masking_key.pack("C*")*((data.size/4) + 1))[0...data.size],"byte")
+					data = (mask_na ^ payload_na).to_s
+				end
+				data
+			end
+		end
+	end
+end
+
+
+
+class String
+
+	def starexp
+		Regexp.new(Regexp.escape(self).gsub('\*','.*?'))
+	end
+
+
+	def starexp?
+		self.include?('*')
+	end
+
+end
+
+
+Signal.trap("USR1") {
+	t1 = Time.new;  GC.start  ;d = Time.new - t1
+	puts "Long GC run:\n        %.3fs"%(d) if DEBUG and d > 0.05
+}
+
+
+#
+# Code is layered, with each layer only communicating with neighbouring layers (e.g. object store never directly talks to em or sockets).
+#
+# Object Store is a singleton responsible for:
+#    * managing objects' lifetime
+#    * dependency tracking and triggering object rebuilds (aka. production)
+#    * tracking of available producers and consumers
+#
+# Client class is responsible for:
+#    * keeping track of clients and their state
+#    * keeping network communication efficient (diffing)
+#
+# Eventmachine main loop is a:
+#    * router between physical world and Client class / instances
+#
+
+
+module SeapigObjectStore
+
+	@@objects_by_id = {} # {id => object}; stores all existing SeapigObjects
+
+	@@producers = {} # {pattern_or_id => {client}}; for assessing spawning possibility
+	@@consumers = {} # {pattern_or_id => {client}}; for assessing spawning need, for assessing holding need
+
+	@@dependents = {}   # {id_depended_on => {id_depending}}; for assessing spawning need, for assessing holding need, for assessing reproduction need
+	@@dependencies = {} # {id_depending => {id_depended_on}}; for updating dependents
+
+	@@queue = []      # [object]; objects in need of production
+	@@producing = {}  # {client => object}; for assessing client busy status
+	@@produced = {}   # {id_being_produced => {version}}; for assessing enqueuing/dequeuing need
+
+
+	def self.consumer_register(pattern_or_id, client)
+		@@consumers[pattern_or_id] = Set.new if not @@consumers[pattern_or_id]
+		@@consumers[pattern_or_id].add(client)
+		self.matching(pattern_or_id, @@producers.merge(@@objects_by_id)).each { |matching_id|
+			@@objects_by_id[matching_id].consumer_register(pattern_or_id, client) if @@objects_by_id[matching_id]
+			self.spawn(matching_id) if not @@objects_by_id[matching_id]
+		}
+	end
+
+
+	def self.producer_register(pattern_or_id, client)
+		@@producers[pattern_or_id] = Set.new if not @@producers[pattern_or_id]
+		@@producers[pattern_or_id].add(client)
+		self.matching(pattern_or_id, @@consumers.merge(@@dependents)).each { |matching_id|
+			@@objects_by_id[matching_id].producer_register(pattern_or_id, client) if @@objects_by_id[matching_id]
+			self.spawn(matching_id) if not @@objects_by_id[matching_id]
+		}
+		self.dequeue(client,nil) if not @@producing[client]
+	end
+
+
+	def self.consumer_unregister(pattern_or_id, client)
+		raise "Unregister without register" if (not @@consumers[pattern_or_id]) or (not @@consumers[pattern_or_id].include?(client))
+		@@consumers[pattern_or_id].delete(client)
+		@@consumers.delete(pattern_or_id) if @@consumers[pattern_or_id].size == 0
+		self.matching(pattern_or_id,@@producers.merge(@@objects_by_id)).each { |matching_id|
+			@@objects_by_id[matching_id].consumer_unregister(pattern_or_id, client) if @@objects_by_id[matching_id]
+			self.despawn(@@objects_by_id[matching_id]) if @@objects_by_id[matching_id] and (not @@objects_by_id[matching_id].alive?) and (not @@dependents[matching_id])
+		}
+	end
+
+
+	def self.producer_unregister(pattern_or_id,client)
+		raise "Unregister without register" if (not @@producers[pattern_or_id]) or (not @@producers[pattern_or_id].include?(client))
+		@@producers[pattern_or_id].delete(client)
+		@@producers.delete(pattern_or_id) if @@producers[pattern_or_id].size == 0
+		self.matching(pattern_or_id,@@consumers.merge(@@dependents)).each { |matching_id|
+			@@objects_by_id[matching_id].producer_unregister(pattern_or_id, client) if @@objects_by_id[matching_id]
+			self.despawn(@@objects_by_id[matching_id]) if @@objects_by_id[matching_id] and (not @@objects_by_id[matching_id].alive?) and (not @@dependents[matching_id])
+		}
+	end
+
+
+	def self.version_get(client,id,version)
+		raise "version_get called on starexp, that doesn't make sense" if id.starexp?
+		return [0,{}] if not @@objects_by_id.has_key?(id)
+		@@objects_by_id[id].version_get(version)
+	end
+
+
+	# data can be one of:
+	# - Hash  => given version corresponds to given data
+	# - false => given version has no data (aka. stall)
+	# - true  => given version exists (data unknown)
+	# - nil   => given version could not be generated (data unknown)
+	def self.version_set(client,id,version,data,requested_version)
+		raise "Update of pattern doesn't make sense" if id.starexp?
+
+		if requested_version != false
+			raise "client not in @@producing" if not @@producing[client]
+			raise "requested_version (%s) not in @@produced[id] (%s)"%[requested_version.inspect,@@produced[id].inspect] if not @@produced[id].include?(requested_version)
+			@@producing.delete(client)
+			@@produced[id].delete(requested_version)  # also on disconnection / unproducer / test
+			@@produced.delete(id) if @@produced[id].size == 0
+		end
+
+		if @@objects_by_id.has_key?(id) or @@dependents[id] or @@consumers.keys.find { |pattern| id =~ pattern.starexp }
+			object = (@@objects_by_id[id] or self.spawn(id))
+			accepted = object.version_set(data, version, requested_version)
+			if accepted
+				puts "Version accepted" if DEBUG
+				(@@dependents[id] or Set.new).each { |dependent_id|
+					raise if not @@objects_by_id.has_key?(dependent_id)
+					next if not (dependent = @@objects_by_id[dependent_id])
+					if dependent.version_needed and dependent.version_needed[id] and version.kind_of?(Integer) and dependent.version_needed[id].kind_of?(Integer) and dependent.version_needed[id] < version
+						dependent.version_needed[id] = version
+						enqueue(dependent)
+					end
+				}
+				if version.kind_of? Hash
+					object.version_needed = {} if (not object.version_needed) or object.version_needed.kind_of?(Integer)
+					old_dependencies = (@@dependencies[id] or Set.new)
+					new_dependencies = (@@dependencies[id] = Set.new(version.keys))
+					(new_dependencies - old_dependencies).each { |added_dependency|
+						object.version_needed[added_dependency] = SeapigObject.version_newer((@@objects_by_id[added_dependency] ? @@objects_by_id[added_dependency].version_latest : 0), (version[added_dependency] or 0))
+						dependent_add(added_dependency, object.id)
+					}
+					(old_dependencies & new_dependencies).each { |kept_dependency|
+						object.version_needed[kept_dependency] = SeapigObject.version_newer((@@objects_by_id[kept_dependency] ? @@objects_by_id[kept_dependency].version_latest : 0), (version[kept_dependency] or 0))
+					}
+					(old_dependencies - new_dependencies).each { |removed_dependency|
+						object.version_needed.delete(removed_dependency)
+						dependent_remove(removed_dependency, object.id)
+					}
+				else
+					object.version_needed = version
+				end
+			end
+			enqueue(object)
+		end
+
+		dequeue(client,nil) if requested_version != false and not @@producing[client]
+	end
+
+
+	def self.cache_get(object_id, key)
+		return nil if not @@objects_by_id.has_key?(object_id)
+		@@objects_by_id[object_id].cache_get(key)
+	end
+
+
+	def self.cache_set(object_id, key, value)
+		return value if not @@objects_by_id.has_key?(object_id)
+		@@objects_by_id[object_id].cache_set(key, value)
+		value
+	end
+
+
+private
+
+
+	class SeapigObject
+
+		attr_reader :id, :versions, :direct_producers, :wildcard_producers
+		attr_accessor :version_needed
+
+		def initialize(id)
+			@id = id
+			@versions = [ [0, {}] ]
+			@direct_consumers = Set.new
+			@wildcard_consumers = {}
+			@direct_producers = Set.new
+			@wildcard_producers = {}
+			@version_needed = nil
+			@cache = []
+		end
+
+
+		def destroy
+			@wildcard_consumers.keys.each { |client|
+				client.object_destroy(@id)
+			}
+		end
+
+
+		def version_get(object_version)
+			@versions.assoc(object_version) or [0,{}]
+		end
+
+
+		def version_set(data,version,requested_version)
+			return false if data == nil
+			return false if not SeapigObject.version_newer?(version_latest, version)
+			@version_needed = version if data == true and ((not @version_needed) or SeapigObject.version_newer?(@version_needed, version))
+			return false if data == true
+			@versions << [version,data]
+			(Set.new(@wildcard_consumers.keys)+@direct_consumers).each { |client| client.object_update(@id, version, data) } if data
+			versions_with_valid_data = 0
+			discard_below = @versions.size - 1
+			while discard_below > 0 and versions_with_valid_data < 1
+				versions_with_valid_data += 1 if @versions[discard_below][1]
+				discard_below -= 1
+			end
+			discard_below.times { @versions.shift }
+			true
+		end
+
+
+		def version_latest
+			return nil if not @versions[-1]
+			@versions[-1][0]
+		end
+
+
+		def self.version_newer?(latest,vb)
+#			return true          if      latest.nil?            and (not vb.nil?)
+#			return false         if (not latest.nil?)           and      vb.nil?
+			return latest < vb   if (not latest.kind_of?(Hash)) and (not vb.kind_of?(Hash))
+			return true          if (not latest.kind_of?(Hash)) and (    vb.kind_of?(Hash))
+			return false         if (    latest.kind_of?(Hash)) and (not vb.kind_of?(Hash))
+			(latest.keys & vb.keys).each { |key|
+				return true if version_newer?(latest[key], vb[key])
+			}
+			return vb.size < latest.size #THINK: is this the right way to go...
+		end
+
+
+		def self.version_newer(va,vb)
+			version_newer?(va,vb) ? vb : va
+		end
+
+
+		def consumer_register(pattern,client)
+			return false if ((not pattern.starexp?) and @direct_consumers.include?(client)) or (pattern.starexp? and @wildcard_consumers[client] and @wildcard_consumers[client].include?(pattern))
+			if pattern.starexp?
+				(@wildcard_consumers[client] ||= Set.new).add(pattern)
+			else
+				@direct_consumers.add(client)
+			end
+			latest_known_version, latest_known_data = @versions.reverse.find { |version,data| data }
+			(Set.new(@wildcard_consumers.keys)+@direct_consumers).each { |client| client.object_update(@id, latest_known_version, latest_known_data) }
+		end
+
+
+		def producer_register(pattern,client)
+			return false if ((not pattern.starexp?) and @direct_producers.include?(client)) or (pattern.starexp? and @wildcard_producers[client] and @wildcard_producers[client].include?(pattern))
+			if pattern.starexp?
+				(@wildcard_producers[client] ||= Set.new).add(pattern)
+			else
+				@direct_producers.add(client)
+			end
+		end
+
+
+		def consumer_unregister(pattern,client)
+			raise "Unregister without register" if (not @direct_consumers.include?(client)) and ((not @wildcard_consumers.has_key?(client)) or (not @wildcard_consumers[client].include?(pattern)))
+			if pattern.starexp?
+				@wildcard_consumers[client].delete(pattern)
+				@wildcard_consumers.delete(client) if @wildcard_consumers[client].size == 0
+			else
+				@direct_consumers.delete(client)
+			end
+		end
+
+
+		def producer_unregister(pattern,client)
+			raise "Unregister without register" if (not @direct_producers.include?(client)) and ((not @wildcard_producers.has_key?(client)) or (not @wildcard_producers[client].include?(pattern)))
+			if pattern.starexp?
+				@wildcard_producers[client].delete(pattern)
+				@wildcard_producers.delete(client) if @wildcard_producers[client].size == 0
+			else
+				@direct_producers.delete(client)
+			end
+		end
+
+
+		def cache_get(key)
+			ret = @cache.assoc(key)
+			puts "Cache "+(ret ? "hit" : "miss") if DEBUG
+			ret and	ret[1]
+		end
+
+
+		def cache_set(key, value)
+			@cache.delete(old_entry) if old_entry = @cache.assoc(key)
+			@cache << [key,value] if OBJECT_CACHE_SIZE > 0
+			@cache = @cache[-OBJECT_CACHE_SIZE..-1] if @cache.size > OBJECT_CACHE_SIZE
+		end
+
+
+		def alive?
+			(@direct_consumers.size > 0 or (@wildcard_consumers.size > 0 and @direct_producers.size > 0))
+		end
+
+
+		def inspect
+			'<SO:%s:%s:%s:%s:%s:%s:%s>'%[@id, @versions.map { |v| v[0] }.inspect,@direct_producers.map(&:id).inspect,@wildcard_producers.keys.map(&:id).inspect,@direct_consumers.map(&:id).inspect,@wildcard_consumers.keys.map(&:id).inspect,@version_needed.inspect]
+		end
+
+	end
+
+
+	def self.matching(pattern,check_against)
+		if pattern.starexp?
+			check_against.each_key.map { |id|
+				id if (not id.starexp?) and (id =~ pattern.starexp)
+			}.compact
+		else
+			(check_against.each_key.find { |id|
+				 (id.starexp? and pattern =~ id.starexp) or ((not id.starexp?) and pattern == id)
+			}) ? [pattern] : []
+		end
+	end
+
+
+	def self.spawn(id)
+		puts "Creating:\n        "+id if DEBUG
+		@@objects_by_id[id] = object = SeapigObject.new(id)
+		@@producers.each_pair.map { |pattern,clients| clients.each { |client| object.producer_register(pattern,client) if pattern.starexp? and (id =~ pattern.starexp) or (id == pattern) } }
+		@@consumers.each_pair.map { |pattern,clients| clients.each { |client| object.consumer_register(pattern,client) if pattern.starexp? and (id =~ pattern.starexp) or (id == pattern) } }
+		enqueue(object)
+		object
+	end
+
+
+	def self.despawn(object)
+		puts "Deleting:\n        "+object.id if DEBUG
+		raise "Despawning object that should stay alive" if object.alive? or @@dependents[object.id]
+		object.destroy
+		(@@dependencies.delete(object.id) or []).each { |dependency_id|
+			dependent_remove(dependency_id, object.id)
+		}
+		@@objects_by_id.delete(object.id)
+	end
+
+
+	def self.enqueue(object)
+		if object.version_needed and object.version_latest == object.version_needed
+			@@queue.delete(object)
+		else
+			return if @@queue.include?(object) or (@@produced[object.id] and @@produced[object.id].include?(object.version_needed))
+			@@queue << object
+			(Set.new(object.direct_producers) + object.wildcard_producers.keys).find { |client|
+				dequeue(client, object) if not @@producing[client]
+			}
+		end
+	end
+
+
+	def self.dequeue(client,object)
+		object = @@queue.find { |candidate_object| candidate_object.direct_producers.include?(client) or candidate_object.wildcard_producers.has_key?(client) } if not object
+		return false if not @@queue.include?(object)
+		version_snapshot = (object.version_needed == nil ? nil : (object.version_needed.kind_of?(Fixnum) ? object.version_needed : object.version_needed.clone))
+		client.object_produce(object.id, version_snapshot)
+		@@queue.delete(object)
+		@@producing[client] = object
+		(@@produced[object.id] ||= Set.new) << version_snapshot
+	end
+
+
+	def self.dependent_add(id, dependent)
+		@@dependents[id] = Set.new if not @@dependents[id]
+		@@dependents[id] << dependent
+		self.matching(id, @@producers).each { |matching_id|
+			self.spawn(matching_id) if not @@objects_by_id[matching_id]
+		}
+	end
+
+
+	def self.dependent_remove(id, dependent)
+		@@dependents[id].delete(dependent)
+		@@dependents.delete(id) if @@dependents[id].size == 0
+		self.despawn(@@objects_by_id[id]) if @@objects_by_id.include?(id) and (not @@objects_by_id[id].alive?) and (not @@dependents[id])
+	end
+
+
+	def self.pp
+		[
+			"Objects:",   @@objects_by_id.values.map { |object| "        %s"%[object.inspect] }.join("\n"),
+			"Queue:",     @@queue.map { |object|                "        %s"%[object.inspect] }.join("\n"),
+			"Producing:", @@producing.map { |client,object|     "        %s - %s"%[client.id,object.id] }.join("\n"),
+			"Produced:",  @@produced.map { |object,versions|    "        %s - %s"%[object,versions.inspect] }.join("\n")
+		].select { |str| str.size > 0 }.join("\n")+"\n"
+	end
+
+
+
+
+
+
+end
+
+
+
+#TODO:
+# * Refactor to have ClientSpace class/module with Clients inside
+
+
+class Client
+
+	attr_reader :produces, :consumes, :socket, :producing, :index, :pong_time
+	attr_accessor :options
+
+	@@clients_by_socket = {}
+	@@count = 0
+
+
+	def self.[](socket)
+		@@clients_by_socket[socket]
+	end
+
+
+	def initialize(socket)
+		@index = @@count += 1
+		puts "Client connected:\n        "+@index.to_s if DEBUG
+		@socket = socket
+		@options = {}
+		@produces = Set.new
+		@consumes = Set.new
+		@versions = {}
+		@producing = nil
+		@@clients_by_socket[socket] = self
+		self.pong
+	end
+
+
+	def id
+		(@options['name'] or "") + ':' + @index.to_s
+	end
+
+
+	def inspect
+		id
+	end
+
+
+	def destroy
+		puts "Client disconnected:\n        "+@index.to_s if DEBUG
+		@@clients_by_socket.delete(@socket)
+		@produces.each { |pattern| SeapigObjectStore.producer_unregister(pattern,self) }
+		@consumes.each { |pattern| SeapigObjectStore.consumer_unregister(pattern,self) }
+		producing = @producing
+		@producing = nil
+		SeapigObjectStore.version_set(self,producing[0],nil,nil,producing[1]) if producing
+	end
+
+
+	def producer_register(pattern, known_version)
+		@produces.add(pattern)
+		SeapigObjectStore.producer_register(pattern, self)
+		SeapigObjectStore.version_set(self, pattern, known_version, true, false) if known_version and not pattern.starexp?
+	end
+
+
+	def producer_unregister(pattern)
+		@produces.delete(pattern)
+		SeapigObjectStore.producer_unregister(pattern, self)
+		if @producing and (pattern.starexp? ? (@producing[0] =~ pattern.starexp) : (@producing[0] == pattern)) #NOTE: overlaping production patterns are not supported
+			producing = @producing
+			@producing = nil
+			SeapigObjectStore.version_set(self,producing[0],nil,nil,producing[1])
+		end
+	end
+
+
+	def consumer_register(pattern, known_version)
+		@consumes.add(pattern)
+		@versions[pattern] = known_version if not pattern.starexp?
+		SeapigObjectStore.consumer_register(pattern, self)
+		gc_versions
+	end
+
+
+	def consumer_unregister(pattern)
+		@consumes.delete(pattern)
+		SeapigObjectStore.consumer_unregister(pattern, self)
+		gc_versions
+	end
+
+
+	def gc_versions
+		@versions.keys.each { |object_id|
+			@versions.delete(object_id) if not (@consumes).find { |pattern|
+				pattern.starexp? and (object_id =~ pattern.starexp) or (pattern == object_id)
+			}
+		}
+	end
+
+
+	def object_update(object_id, object_version, object_data)
+		#THINK: should we propagate stalls to clients?
+		return if object_version == 0 or object_version == @versions[object_id]
+		old_version, old_data = SeapigObjectStore.version_get(self,object_id,(@versions[object_id] or 0))
+		data = if old_version == 0
+			       { "value" => object_data }
+		       else
+			       diff = SeapigObjectStore.cache_get(object_id,[:diff,old_version,object_version])
+			       diff = SeapigObjectStore.cache_set(object_id,[:diff,old_version,object_version],JsonDiff.generate(old_data, object_data)) if not diff
+			       { "patch" => diff }
+		       end
+
+		json = Oj.dump({
+					 "action" => 'object-update',
+					 "id" => object_id,
+					 "old_version" => old_version,
+					 "new_version" => object_version,
+				 }.merge(data))
+		puts "Sending:\n        %8iB  %s to %s"%[json.size, object_id, id] if DEBUG
+		@versions[object_id] = object_version
+		@socket.send json
+	end
+
+
+	def object_destroy(object_id)
+		@socket.send Oj.dump("action" => 'object-destroy', "id" => object_id)
+	end
+
+
+	def object_patch(object_id, patch, value, from_version, to_version)
+		raise "patching wildcard object. no." if object_id.starexp?
+		requested_object_id, requested_version = @producing
+		if requested_object_id == object_id
+			@producing = nil
+		else
+			requested_version = false
+		end
+		new_version = to_version
+
+		new_data = if patch
+				   object_version, object_data = SeapigObjectStore.version_get(self,object_id,from_version)
+				   print "Patching:\n        version: "+object_version.inspect+"\n        from_version: "+from_version.inspect+"\n        to_version: "+to_version.inspect+"\n        patch_size: "+(patch and patch.size.to_s or "nil")+"\n        --> "  if DEBUG
+				   if from_version == object_version
+					   puts 'clean' if DEBUG
+					   new_data = Oj.load(Oj.dump(object_data))
+					   begin
+						   Hana::Patch.new(patch).apply(new_data) if patch
+					   rescue Exception => e
+						   puts "Patching failed!\n        Old object: "+object_data.inspect+"\n        Patch: "+patch.inspect if DEBUG
+						   raise e
+					   end
+					   new_data
+				   else
+					   puts "can't update object, couldn't find base version" if DEBUG
+					   nil
+				   end
+			   elsif value != nil
+				   print "Setting:\n        version: "+object_version.inspect+"\n        from_version: "+from_version.inspect+"\n        to_version: "+to_version.inspect+"\n        value_size: "+(value.inspect.size.to_s)+"\n"  if DEBUG
+				   value
+			   else
+				   nil
+			   end
+
+		SeapigObjectStore.version_set(self,object_id,new_version,new_data,requested_version)
+	end
+
+
+	def object_produce(object_id, object_version)
+		raise "Can't produce a wildcard object" if object_id.starexp?
+		raise "Client already producing something (producing: %s, trying to assign: %s)"%[@producing.inspect, [object_id,object_version].inspect] if @producing
+		raise "Can't produce that pattern: "+@produces.inspect+" "+object_id.inspect if not @produces.find { |pattern| object_id =~ pattern.starexp }
+		puts "Assigning:\n        "+object_id+':'+object_version.inspect+'    to: '+self.id if DEBUG
+		@socket.send Oj.dump("action" => 'object-produce', "id" => object_id, "version"=>object_version)
+		@producing = [object_id, object_version]
+	end
+
+
+	def pong
+		@pong_time = Time.new
+	end
+
+
+	def self.send_pings
+		@@clients_by_socket.keys.each { |socket| socket.ping }
+	end
+
+
+	def self.send_heartbeats
+		@@clients_by_socket.each_pair { |socket,client| socket.send Oj.dump('action' => 'heartbeat') if client.options['heartbeat'] }
+	end
+
+
+	def self.check_ping_timeouts
+		@@clients_by_socket.each_pair { |socket,client| socket.close if Time.new - client.pong_time > 60 }
+	end
+
+
+	def self.pp
+		"Clients:\n"+@@clients_by_socket.values.map { |client| "        %-20s    produces:%s    consumes:%s"%[client.id,client.produces.to_a,client.consumes.to_a] }.join("\n")+"\n"
+	end
+end
+
+
+class InternalClient
+
+	def self.produce
+	end
+
+	def initialize
+		SeapigObjectStore.producer_register("SeapigServer::Objects", self)
+	end
+
+	def object_produce(object_id, object_version)
+		objects =
+		SeapigObjectStore.version_set(object_id,new_version,objects,object_version)
+	end
+
+end
+
+
+#TODO:
+# * change protocol to use "pattern" instead of "id"
+# * change "object-patch" to something nicer
+
+processing_times = []
+processing_times_sum = 0
+
+EM.run {
+
+
+	WebSocket::EventMachine::Server.start(host: HOST, port: PORT) { |client_socket|
+
+		client_socket.onmessage { |message|
+			begin
+				started_at = Time.new
+				client = Client[client_socket]
+				message = Oj.load message
+				puts "-"*80 + ' ' + Time.new.to_s if DEBUG
+				print "Message:\n        from: %-20s\n        action: %-30s\n        param: %-50s "%[client.id, message['action'], Oj.dump(message.select { |k,v| ['pattern','id','options'].include?(k) })] if DEBUG
+				puts if DEBUG
+				object_id = message['id'] if message['id']
+				case message['action']
+				when 'object-producer-register'
+					fail unless message['pattern']
+					client.producer_register(message['pattern'],message['known-version'])
+				when 'object-producer-unregister'
+					fail unless message['pattern']
+					client.producer_unregister(message['pattern'])
+				when 'object-patch'
+					fail unless message['id']
+					client.object_patch(object_id,message['patch'], message['value'], message['old_version'], message['new_version'])
+				when 'object-consumer-register'
+					fail unless message['id']
+					client.consumer_register(object_id,message['known-version'])
+				when 'object-consumer-unregister'
+					fail unless message['id']
+					client.consumer_unregister(object_id)
+				when 'client-options-set'
+					fail unless message['options']
+					client.options = message['options']
+				else
+					raise 'WTF, got message with action: ' + message['action'].inspect
+				end
+				processing_times << (Time.new.to_f - started_at.to_f)
+				processing_times_sum += processing_times[-1]
+				if DEBUG
+					puts Client.pp
+					puts SeapigObjectStore.pp
+					puts "Processing:\n        time: %.3fs\n        count: %i\n        average: %.3fs\n        total: %.3fs"%[processing_times[-1], processing_times.size, processing_times_sum / processing_times.size, processing_times_sum]
+				end
+				puts "message:%3i  t:%.3fs  Σt:%.3fs  t̅:%.3fs"%[processing_times.size, processing_times[-1],   processing_times_sum, processing_times_sum / processing_times.size,] if INFO and not DEBUG
+			rescue => e
+				puts "Message processing error:\n        "
+				p e
+				e.backtrace.each { |line| puts line }
+				raise
+			end
+		}
+
+
+		client_socket.onopen { Client.new(client_socket) }
+		client_socket.onclose { Client[client_socket].destroy if Client[client_socket] }
+		client_socket.onpong { Client[client_socket].pong }
+	}
+
+	puts "Listening on %s:%s"%[HOST,PORT] if INFO or DEBUG
+        Socket.open(:UNIX, :DGRAM)  { |s| s.connect(Socket.pack_sockaddr_un(ENV['NOTIFY_SOCKET'])); s.sendmsg "READY=1" } if ENV['NOTIFY_SOCKET']
+
+	EM.add_periodic_timer(10) { Client.send_pings }
+	EM.add_periodic_timer(10) { Client.send_heartbeats }
+	EM.add_periodic_timer(10) { Client.check_ping_timeouts }
+
+	EM.add_periodic_timer(1) {
+		now = Time.new
+		puts "CPU time used: %7.3f%%"%[(processing_times_sum-$last_processing_times_sum)*100.0/(now - $last_cpu_time)] if $last_cpu_time and DEBUG
+		$last_cpu_time = now
+		$last_processing_times_sum = processing_times_sum
+	}
+
+
+	close_reader, close_writer = IO.pipe
+
+	EM.watch(close_reader) { |connection|
+		connection.notify_readable = true
+		connection.define_singleton_method(:notify_readable) do
+			puts "Shutting down" if INFO or DEBUG
+			exit
+		end
+	}
+
+	Signal.trap("INT") {
+		puts "SIGINT received, scheduling exit." if DEBUG
+		close_writer.write('.')
+	}
+
+}

data/lib/seapig/version.rb

@@ -1,3 +1,3 @@
 module Seapig
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: seapig-server
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - yunta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-06 00:00:00.000000000 Z
+date: 2017-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: websocket-eventmachine-server
@@ -92,6 +92,7 @@ files:
 - README.rdoc
 - Rakefile
 - bin/seapig-server
+- bin/seapig-server-intro
 - lib/seapig-server.rb
 - lib/seapig/version.rb
 - test/dummy/log/development.log