dp_stm_map 0.0.2

data/lib/dp_stm_map/Manager.rb

@@ -0,0 +1,370 @@
+# dp_stm_map - Distributed and Persistent Software Transaction Map
+# Copyright (C) 2013  Dragan Milic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+require 'socket'
+require 'thread'
+require 'set'
+require 'fileutils'
+require 'threadsafe-lru'
+
+module DpStmMap
+
+  class ManagerShutdown < StandardError
+  end
+
+
+
+  class Manager
+
+    def initialize port, store_dir
+      @port=port
+      @state=:not_running
+
+
+      FileUtils.mkdir_p(store_dir) unless File.exist? store_dir
+
+      @transaction_log=TransactionLog.new("%s/log" % store_dir)
+      reference_counts=ReferenceCounts.new("%s/refcounts" % store_dir)
+      current_values=CurrentValues.new("%s/current_values" % store_dir)
+      @tx_manager=ClientTransactionManager.new(current_values, reference_counts, @transaction_log)
+    end
+
+    def port
+      @server_socket.addr[1]
+    end
+
+
+    def start
+      @server_socket=TCPServer.new @port
+      @accept_thread=Thread.new do
+        begin
+          active_threads=[]
+          loop do
+            client_socket=@server_socket.accept
+            client_socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+
+            active_threads << Thread.new do
+
+              begin
+                transport=SocketTransport.new(client_socket)
+                client_handler=ClientHandler.new transport, @tx_manager
+                msg=transport.next_message
+
+                raise "client did not send ClientHelloMessage" unless ClientHelloMessage === msg
+
+                transport.send_message ServerHelloMessage.new
+
+
+                @transaction_log.add_listener(msg.last_transaction_sequence) do |seq, tx|
+                  client_handler.handle TransactionMessage.new(seq, *tx)
+                end
+
+                loop do
+                  msg=transport.next_message
+                  client_handler.handle msg
+                end
+
+              rescue ManagerShutdown => e
+
+
+              rescue => e
+                # puts "Error during processing: #{$!}"
+                # puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+
+              ensure
+                # puts "closed socket to client"
+                client_socket.close
+              end
+            end
+          end
+
+        rescue ManagerShutdown => e
+        ensure
+          @server_socket.close unless @server_socket.closed?
+          active_threads.each do |thread|
+            thread.raise ManagerShutdown
+            thread.join
+          end
+
+
+
+        end
+
+
+      end
+    end
+
+
+    def stop
+      @accept_thread.raise ManagerShutdown
+
+      begin
+        @accept_thread.join
+      rescue => e
+      end
+
+
+    end
+  end
+
+
+  class CurrentValues
+
+    def initialize current_values_dir
+      @current_values_dir=current_values_dir
+      FileUtils.mkdir_p(current_values_dir) unless File.exist?(current_values_dir)
+      @cache=ThreadSafeLru::LruCache.new 9000
+    end
+
+    def file_name key
+      "#{@current_values_dir}/#{key}"
+    end
+
+    def [] k
+      @cache.get(k) do |key|
+        File.exist?(file_name(key)) ? File.open(file_name(key),"r"){|f| f.read()} : nil
+      end
+    end
+
+    def []= key, value
+      if value
+        File.open(file_name(key),"w"){|f| f.write(value)}
+      else
+        File.unlink file_name(key) if File.exist? file_name(key)
+      end
+      @cache.drop(key)
+    end
+
+  end
+
+
+
+  class ReferenceCounts
+    def initialize references_dir
+      @references_dir=references_dir
+      FileUtils.mkdir_p(references_dir) unless File.exist?(references_dir)
+      @cache=ThreadSafeLru::LruCache.new 9000
+    end
+
+    def file_name ref
+      "#{@references_dir}/#{ref}"
+    end
+
+    def add_reference r
+      count=0
+      @cache.get(r) do |ref|
+        count=File.open(file_name(ref),"r") {|f| f.read().to_i} if File.exist? file_name(ref)
+      end
+      File.open(file_name(r),"w") {|f| f.write(count+1)}
+      @cache.drop(r)
+    end
+
+    def remove_reference ref
+      count=File.open(file_name(ref),"r") {|f| f.read().to_i}
+      File.open(file_name(ref),"w") {|f| f.write(count-1)}
+      @cache.drop(ref)
+    end
+
+    def has_references? r
+      count=0
+      @cache.get(r) do |ref|
+        count=File.open(file_name(ref),"r") {|f| f.read().to_i} if File.exist? file_name(ref)
+      end
+      count > 0
+    end
+
+  end
+
+  class TransactionLog
+
+    def initialize log_dir
+      @log_dir=log_dir
+
+      FileUtils.mkdir_p(log_dir) unless File.exist?(log_dir)
+
+      unless File.exist?("#{log_dir}/last_id.txt")
+        File.open("#{log_dir}/last_id.txt","w") {|f| f.write("0")}
+        @last_id=0
+      else
+        @last_id=File.open("#{log_dir}/last_id.txt","r") {|f| f.read().to_i}
+      end
+
+      @mutex=Mutex.new
+      @change=ConditionVariable.new
+    end
+
+
+    def read_tx id
+      JSON::parse(File.open("#{@log_dir}/#{id}.json","r") {|f| f.read})
+    end
+
+    def write_tx id, tx
+
+      File.open("#{@log_dir}/#{id}.json","w") {|f| f.write(tx.to_json)}
+    end
+
+
+
+    def add_listener current_state, &block
+      Thread.new do
+        begin
+          loop do
+            arg=@mutex.synchronize do
+              until current_state < last_id
+                @change.wait(@mutex)
+              end
+              current_state+=1
+              [current_state, read_tx(current_state)]
+            end
+            yield arg
+          end
+
+
+        rescue => e
+          # puts "Error during processing: #{$!}"
+          # puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
+        end
+      end
+    end
+
+
+    def last_id
+      @last_id
+    end
+
+    def store_transaction references_to_change, values_to_add, values_to_remove
+      @mutex.synchronize do
+        txid=last_id+1
+        write_tx txid, [references_to_change, values_to_add, values_to_remove]
+        File.open("#{@log_dir}/last_id.txt","w") {|f| f.write(txid.to_s)}
+        @change.broadcast
+        @last_id=txid
+        txid
+      end
+    end
+
+
+  end
+
+  class SocketTransport
+    def initialize socket
+      @socket=socket
+    end
+
+    def next_message
+      length=@socket.read(8).unpack("Q>")[0]
+      JsonMessage::deserialize(@socket.read(length))
+    end
+
+    def send_message msg
+
+      data=msg.serialize
+      @socket.write([data.bytesize].pack("Q>"))
+      @socket.write(data)
+      @socket.flush
+    end
+
+    def close
+      @socket.close
+    end
+
+
+  end
+
+
+  class ClientHandler
+
+    def initialize client_transport, transaction_handler
+      @client_transport=client_transport
+      @transaction_handler=transaction_handler
+      @mutex=Mutex.new
+    end
+
+    def handle message
+      @mutex.synchronize do
+        if TransactionMessage === message
+          @client_transport.send_message message
+        end
+        if ClientTransactionMessage === message
+          begin
+            sequence=@transaction_handler.apply_transaction message.transitions, message.new_content
+            @client_transport.send_message ClientTransactionSuccessfulMessage.new message.transaction_id, sequence
+          rescue StaleTransactionError => e
+            @client_transport.send_message ClientTransactionFailedMessage.new message.transaction_id, e.message
+          end
+        end
+      end
+    end
+
+  end
+
+
+  class StaleTransactionError < StandardError
+  end
+
+  class ClientTransactionManager
+
+    def initialize current_values, reference_counts, transaction_log
+      @current_values=current_values
+      @reference_counts=reference_counts
+      @transaction_log=transaction_log
+    end
+
+
+    def apply_transaction transitions, new_values
+
+      values_to_remove=Set.new
+      values_to_add={}
+      references_to_change={}
+
+      transitions.each do |k,(old,new)|
+        current=@current_values[k]
+        raise StaleTransactionError, "transaction changing stale state" if (current != old)
+
+        next if old == new
+
+        if new && !@reference_counts.has_references?(new)
+          unless new_values.has_key? new
+            raise StaleTransactionError, "referencing not existing data"
+          else
+            values_to_add[new]=new_values[new]
+          end
+
+        end
+      end
+
+      transitions.each do |k,(old,new)|
+        if (old != new)
+          values_to_remove << old
+          references_to_change[k]=new
+          @current_values[k]=new
+          @reference_counts.add_reference(new) if new
+          @reference_counts.remove_reference(old) if old
+        end
+      end
+
+      values_to_remove=values_to_remove.select {|v| v && !@reference_counts.has_references?(v)}
+
+      # unless references_to_change.empty?
+      @transaction_log.store_transaction references_to_change, values_to_add, values_to_remove.to_set.to_a
+      # end
+
+
+
+    end
+
+  end
+
+
+
+end

data/lib/dp_stm_map/Message.rb

@@ -0,0 +1,126 @@
+# dp_stm_map - Distributed and Persistent Software Transaction Map
+# Copyright (C) 2013  Dragan Milic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+require 'json'
+
+module DpStmMap
+
+  class JsonMessage
+
+    def self.deserialize string
+      # return YAML::load(string)
+
+      json=JSON.parse(string)
+      type=json.delete('type')
+      cls=class_from_string(type)
+      obj=cls.new
+      obj.from_hash! json
+      obj
+    end
+
+    def self.class_from_string(str)
+      str.split('::').inject(Object) do |mod, class_name|
+        mod.const_get(class_name)
+      end
+    end
+
+    def serialize
+        # return YAML::dump(self)
+        hash = {}
+        self.instance_variables.each do |var|
+            hash[var] = self.instance_variable_get var
+        end
+        hash['type']=self.class.name
+        hash.to_json
+    end
+
+    def from_hash! hash
+        hash.each do |var, val|
+            self.instance_variable_set var, val
+        end
+    end
+
+  end
+
+
+
+  class ClientHelloMessage < JsonMessage
+
+    def initialize last_transaction_sequence=nil
+      @last_transaction_sequence=last_transaction_sequence
+    end
+
+    attr_accessor :last_transaction_sequence
+
+  end
+
+  class ServerHelloMessage < JsonMessage
+  end
+
+  class ClientTransactionOutcomeMessage < JsonMessage
+    def initialize transaction_id
+      @transaction_id=transaction_id
+    end
+    attr_accessor :transaction_id
+  end
+
+  class ClientTransactionSuccessfulMessage < ClientTransactionOutcomeMessage
+
+    def initialize transaction_id=nil, transaction_sequence=nil
+      super transaction_id
+      @transaction_sequence=transaction_sequence
+    end
+
+    attr_accessor :transaction_sequence
+
+  end
+
+
+  class ClientTransactionFailedMessage < ClientTransactionOutcomeMessage
+
+    def initialize transaction_id=nil, reason=nil
+      super transaction_id
+      @reason=reason
+    end
+
+    attr_accessor :reason
+  end
+
+  class ClientTransactionMessage < JsonMessage
+    def initialize transaction_id=nil, transitions=nil, new_content=nil
+      @transaction_id=transaction_id
+      @transitions=transitions
+      @new_content=new_content
+    end
+
+    attr_accessor :transaction_id
+    attr_accessor :transitions
+    attr_accessor :new_content
+  end
+
+  class TransactionMessage < JsonMessage
+    def initialize transaction_sequence=nil, transitions=nil, new_content=nil, delete_content=nil
+      @transaction_sequence=transaction_sequence
+      @transitions=transitions
+      @new_content=new_content
+      @delete_content=delete_content
+    end
+
+    attr_accessor :transaction_sequence
+    attr_accessor :transitions
+    attr_accessor :new_content
+    attr_accessor :delete_content
+
+  end
+
+end

data/lib/dp_stm_map/ObjectStore.rb

@@ -0,0 +1,99 @@
+# dp_stm_map - Distributed and Persistent Software Transaction Map
+# Copyright (C) 2013  Dragan Milic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+require 'yaml'
+
+module DpStmMap
+
+  class ObjectTransactionWrapper
+    def initialize tx
+      @tx=tx
+    end
+
+    def []= domain, key, value
+      @tx[domain_key(domain,key)]=value == nil ? nil : YAML::dump(value)
+    end
+
+    def [] domain, key
+      raw=@tx[domain_key(domain,key)]
+      if (raw == nil)
+        nil
+      else
+        YAML::load(raw)
+      end
+    end
+
+    def has_key? domain, key
+      @tx.has_key?(domain_key(domain,key))
+    end
+
+    def domain_key domain, key
+      "%s:%s" % [domain.to_s, key.to_s]
+    end
+
+  end
+
+  class ObjectStore
+
+    def initialize stm_map
+      @stm_map=stm_map
+    end
+
+
+    def on_atomic
+      @stm_map.on_atomic do |changes|
+        yield changes_to_object_changes changes
+      end
+    end
+
+    def atomic timeout=nil
+      @stm_map.atomic(timeout) do |stx|
+        yield ObjectTransactionWrapper.new(stx)
+      end
+    end
+
+    def atomic_read
+      @stm_map.atomic_read do |stx|
+        yield ObjectTransactionWrapper.new(stx)
+      end
+    end
+
+    def validate_atomic
+      @stm_map.validate_atomic do |changes|
+        yield changes_to_object_changes changes
+      end
+    end
+
+    private
+
+
+    def changes_to_object_changes changes
+      changes.inject({}) {|h, (k,v)| h[string_to_domain_key(k)]=[deserialize_object(v[0]),deserialize_object(v[1])]; h}
+    end
+
+
+    def string_to_domain_key str
+      parts=str.split(/:/)
+      [parts[0].to_sym, parts[1]]
+    end
+
+    def deserialize_object str
+      str == nil ? nil : YAML::load(str)
+    end
+
+  end
+
+
+
+
+end

data/lib/dp_stm_map/version.rb

@@ -0,0 +1,16 @@
+# dp_stm_map - Distributed and Persistent Software Transaction Map
+# Copyright (C) 2013  Dragan Milic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+module DpStmMap
+  VERSION = "0.0.2"
+end

data/lib/dp_stm_map.rb

@@ -0,0 +1,20 @@
+# dp_stm_map - Distributed and Persistent Software Transaction Map
+# Copyright (C) 2013  Dragan Milic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+require "dp_stm_map/version"
+require "dp_stm_map/InMemoryStmMap"
+require "dp_stm_map/Manager"
+require "dp_stm_map/Message"
+require "dp_stm_map/Client"
+require "dp_stm_map/ClientLocalStore"
+require "dp_stm_map/ObjectStore"