dp_stm_map 0.0.2

data/README.md

@@ -0,0 +1,115 @@
+# DpStmMap
+
+Implementation of a distributed and persistent Map with Software Transactional Memory (STM) semantics.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'dp_stm_map'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dp_stm_map
+
+## Comprehensive defintion
+
+  Implementation of a distributed and persistent Map with Software Transactional Memory (STM) semantics.
+
+## What is it?
+
+### Implementation of a hash map (dp_stm_map) with ACID transactional updates.
+
+### Offers a subset of map (Hash) operations
+  
+  :[], :[]= and :has_key?
+
+### Persistent
+  
+  Unlike data stored in the Hash, data stored in dp_stm_map will survive a restart of RVM.
+
+### Distributed
+  
+  More than one RVM can connect to a central transaction manager (dp_stm_manager) that coordinates transactions for the same dp_stm_map. Every RVM can have its own local view of the current state of the map, making reading operations of dp_stm_map very scalable.
+
+## What is so special about it?
+  
+  Every RVM will receive notification on changes in the map. This makes dp_stm_map great foundation for 
+  a distributed Event-driven architecture: Think of one RVM handling web traffic, another indexing persisted state and offering RESTful search service.
+
+## Usage
+
+### Manager
+  to start the central transaction manager execute:
+
+  $ dp_stm_manager.rb -s <storage directory> -p <port>
+
+### Client
+
+  require 'dp_stm_map'
+
+  client=DpStmMap::DistributedPersistentStmMap.new host,port,'storage'
+  client.start  
+  
+#### Updating map within a transaction
+
+  client.atomic do |tx|
+    tx['key'] = 'value'
+  end
+
+#### Read only transactions
+
+  value=client.atomic_read{ |tx| tx['key'] }
+
+#### Adding a transaction listener
+
+  client.on_atomic do |change|
+    # change is map containing all value transitions of one transaction:
+    # e.g. {'key1' => [nil,'value1'], 'key2' => ['old_value_2','new_value2']}
+    change.each_pair do | k, (old_value, new_value) |
+      update_index k, new_value
+    end
+  end
+
+#### Adding a transaction validator - will be executed before transaction is passed on to transaction manager
+
+  client.validate_atomic do |change|
+    # change is map containing all value transitions of one transaction:
+    # e.g. {'key1' => [nil,'value1'], 'key2' => ['old_value_2','new_value2']}
+    change.each_pair do | k, (old_value, new_value) |
+
+      # exception will abort (roll back) the transaction and will be raised as result of executing :atomic method
+      raise "new value is invalid" if new_value != 'valid_new_value'
+    end
+  end
+
+### Using object store wrapper (stores Ruby objects in dp_stm_map by serializing as YAML strings)
+
+  object_client=DpStmMap::ObjectStore.new client
+
+
+  # all transaction, notification and validation methods from dp_stm_map are available to ObjectStore
+
+  object_client.atomic do |tx|
+    user=User.new
+    user.username="admin"
+    tx[:user, "admin"]=user
+  end
+
+  user=object_client.atomic_read { |tx| tx[:user, "admin"] }
+
+  object_client.validate_atomic do |change|
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/dp_map_manager.rb

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# dp_stm_map - Distributed and Persistent Software Transaction Map
+# Copyright (C) 2013  Dragan Milic
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+require 'dp_stm_map'
+require 'optparse'
+
+
+port=0
+store_dir=nil
+OptionParser.new do |opts|
+  opts.banner == "Usage: dp_map_manager.rb -p <port>"
+  opts.on("-p N", Integer, "TCP port number where server should accept conntections") do |p|
+    port=p
+  end
+
+  opts.on("-s DIR", "Directory for storage") do |dir|
+    store_dir=dir
+  end
+
+end.parse!
+
+server=DpStmMap::Manager.new port, store_dir
+
+server.start
+puts "Manager started at port #{server.port}"
+STDOUT.flush
+
+$stdin.each do |line|
+  if /^quit/.match line
+    server.stop
+    puts "Manager shut down"
+    STDOUT.flush
+    exit(0)
+  end
+end

data/cucumber.yml

@@ -0,0 +1,2 @@
+autotest: features -r features --format pretty --color
+autotest-all: features -r features --format pretty --color

data/dp_stm_map.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dp_stm_map/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dp_stm_map"
+  spec.version       = DpStmMap::VERSION
+  spec.authors       = ["Dragan Milic"]
+  spec.email         = ["dragan@netice9.com"]
+  spec.description   = %q{distributed and persistent software transaction memory map}
+  spec.summary       = %q{distributed and persistent software transaction memory map}
+  spec.homepage      = ""
+  spec.license       = "GPLv3"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'autotest'
+  spec.add_development_dependency 'cucumber'
+  spec.add_development_dependency 'aruba'
+  spec.add_dependency 'threadsafe-lru'
+  spec.add_dependency 'xray'
+end

data/features/client_reconnect.feature

@@ -0,0 +1,13 @@
+Feature: client reconnect
+
+  @announce
+  @announce-stderr
+  Scenario: Client should reconnect to the server when the server shuts down
+    Given I start dp_map_manager on port 31331 and store state for key "admin"
+    And I initialize first client connecting to port 31331
+    And I wait for 200ms
+    When I shut down dp_map_manager
+    And I start dp_map_manager on port 31331
+    And I wait for 700ms
+    Then the first client should have state for key "admin"
+    And the first client should be able to change state for key "admin"

data/features/persistence.feature

@@ -0,0 +1,12 @@
+Feature: persistence
+
+  @announce
+  @announce-stderr
+  Scenario: state remains persistent between server restarts
+    Given I start dp_map_manager on port 31331 and store state for key "admin"
+    And I shut down dp_map_manager
+    When I start server on port 31331
+    When I start second client that connects to port 31331
+    And I wait for 200ms
+    Then second client should have data for key "admin"
+    

data/features/replication.feature

@@ -0,0 +1,9 @@
+Feature: replication
+
+  Scenario: replication of state between two clients
+    Given I start dp_map_manager on port 31331
+    And I initialize two clients that connect to port 31331
+    When first client stores data for key "admin"
+    And I wait for 100ms
+    Then second client should have data for key "admin"
+    

data/features/running_manager.feature

@@ -0,0 +1,19 @@
+Feature: running manager
+
+  @announce
+  @announce-stderr
+  Scenario: starting and stopping server
+    Given I run `dp_map_manager.rb -p 31337 -s store` interactively
+    And I wait for stdout to contain "Manager started"    
+    When I type "quit"
+    And I wait for stdout to contain "Manager shut down"
+    Then the exit status should be 0
+
+  @announce
+  @announce-stderr
+ Scenario: configuring port
+   When I start dp_map_manager on port 31331    
+   Then server should be listening at port 31331
+
+
+

data/features/step_definitions/client_reconnect_steps.rb

@@ -0,0 +1,28 @@
+Given(/^I initialize first client connecting to port (\d+)$/) do |port|
+  @client1=create_client('localhost', port.to_i)
+  @client1.start
+  @client2=create_client('localhost', port.to_i)
+  @client2.start
+  sleep 0.2
+  @client1.atomic do |tx|
+    tx['admin']='other_value'
+  end
+end
+
+Then(/^the first client should have state for key "(.*?)"$/) do |key|
+  @client1.atomic_read do |tx|
+    tx.should have_key(key)
+  end
+end
+
+Then(/^the first client should be able to change state for key "(.*?)"$/) do |key|
+  @client1.atomic do |tx|
+    tx[key]='other_value'
+  end
+  sleep 0.2
+  puts "changed!"
+  @client1.atomic_read do |tx|
+    tx[key].should == 'other_value'
+  end
+
+end

data/features/step_definitions/persistence_steps.rb

@@ -0,0 +1,49 @@
+Given(/^I start dp_map_manager on port (\d+) and store state for key "(.*?)"$/) do |port, key|
+  run_interactive(unescape("dp_map_manager.rb -p #{port} -s store"))
+  wait_for_output("Manager started at port #{port}")
+  @server_port=port.to_i
+
+  @client1=create_client('localhost', @server_port)
+  @client1.start  
+  @client1.atomic do |tx|
+    tx[key]='some value'
+  end
+
+  # @client1_socket=TCPSocket.new 'localhost', port.to_i
+  # send_message(@client1_socket, ClientHelloMessage.new(0))
+  # expect_message(@client1_socket, ServerHelloMessage)
+  # send_message(@client1_socket, ClientTransactionMessage.new('tx1',{key => [nil,'abc']},{'abc' => 'some value'}))
+  # success=expect_message(@client1_socket, ClientTransactionSuccessfulMessage)
+  # success.transaction_sequence.should_not be_nil
+  # success.transaction_id.should_not be_nil
+  # tx=expect_message(@client1_socket, TransactionMessage)
+  # tx.transaction_sequence.should == 1
+
+end
+
+Given(/^I shut down dp_map_manager$/) do
+  type('quit')
+  assert_exit_status(0)
+end
+
+When(/^I start server on port (\d+)$/) do |port|
+  @server_port=port.to_i
+  run_interactive(unescape("dp_map_manager.rb -p #{port} -s store"))
+  wait_for_output("Manager started at port #{port}")
+end
+
+When(/^I start second client that connects to port (\d+)$/) do |port|
+  @client2_socket=TCPSocket.new 'localhost', port
+  send_message(@client2_socket, ClientHelloMessage.new(0))
+  expect_message(@client2_socket, ServerHelloMessage)
+end
+
+Then(/^second client should have data for key "(.*?)"$/) do |key|
+  msg=expect_message(@client2_socket, TransactionMessage)
+  msg.transitions.should have_key(key)
+end
+
+
+When(/^I wait for (\d+)ms$/) do |ms|
+  sleep ms.to_f/1000.0
+end

data/features/step_definitions/replication_steps.rb

@@ -0,0 +1,15 @@
+Given(/^I initialize two clients that connect to port (\d+)$/) do |port|
+  @client1_socket=TCPSocket.new 'localhost', port.to_i
+  send_message(@client1_socket, ClientHelloMessage.new(0))
+  expect_message(@client1_socket, ServerHelloMessage)
+
+  @client2_socket=TCPSocket.new 'localhost', port.to_i
+  send_message(@client2_socket, ClientHelloMessage.new(0))
+  expect_message(@client2_socket, ServerHelloMessage)
+end
+
+When(/^first client stores data for key "(.*?)"$/) do |key|
+  send_message(@client1_socket, ClientTransactionMessage.new('tx1',{key => [nil,'abc']},{'abc' => 'some value'}))
+  expect_message(@client1_socket, ClientTransactionSuccessfulMessage)
+  expect_message(@client1_socket, TransactionMessage)
+end

data/features/step_definitions/running_server_steps.rb

@@ -0,0 +1,10 @@
+When(/^I start dp_map_manager on port (\d+)$/) do |port|
+  @server_port=port.to_i
+  run_interactive(unescape("dp_map_manager.rb -p #{port} -s store"))
+  wait_for_output("Manager started")
+end
+
+Then(/^server should be listening at port (\d+)$/) do |port_s|
+  TCPSocket.new 'localhost', port_s.to_i
+end
+

data/features/step_definitions/transaction_fail_steps.rb

@@ -0,0 +1,11 @@
+Given(/^client attempts to perform transaction that updates stale value$/) do
+  @client1_socket=TCPSocket.new 'localhost', @server_port
+  send_message(@client1_socket, ClientHelloMessage.new(0))
+  expect_message(@client1_socket, ServerHelloMessage)
+  send_message(@client1_socket, ClientTransactionMessage.new('tx1',{'a' => ['def','abc']},{'abc' => 'some value'}))  
+end
+
+Then(/^client should be notified that transaction has failed$/) do
+  expect_message(@client1_socket, ClientTransactionFailedMessage)
+end
+

data/features/support/env.rb

@@ -0,0 +1,80 @@
+$LOAD_PATH << File.expand_path('../../../lib',__FILE__)
+require 'aruba/cucumber'
+require 'dp_stm_map'
+require 'tmpdir'
+require 'timeout'
+
+include DpStmMap
+
+
+def wait_for_output expected
+  Timeout::timeout(exit_timeout) do
+    loop do
+      break if assert_partial_output_interactive(expected)
+      sleep 0.1
+    end
+  end
+end
+
+
+
+Before do
+  @tmpdirs=[]
+  @clients=[]
+
+end
+
+After do
+  # FileUtils.rm_rf @client_temp
+  @tmpdirs.each do |tmpdir|
+    FileUtils.rm_rf tmpdir
+  end
+
+  @clients.each do |client|
+    client.stop
+  end
+end
+
+After do
+  begin
+    type('quit')
+    assert_exit_status(0)
+  rescue
+
+  end
+end
+
+
+def make_temp_dir
+  tmpdir=Dir.mktmpdir
+  @tmpdirs << tmpdir
+  tmpdir
+end
+
+
+def send_message socket, message
+  yaml=message.serialize
+  socket.write([yaml.bytesize].pack("Q>"))
+  socket.write(yaml)
+  socket.flush
+end
+
+def expect_message socket, type
+  timeout(3) do
+    size=socket.read(8).unpack("Q>")[0]
+    message=JsonMessage::deserialize(socket.read(size))
+
+    # puts "message: #{message}"
+
+    message.should be_a type
+    message
+  end
+end
+
+
+
+def create_client host, port
+  client=DistributedPersistentStmMap.new host,port,make_temp_dir   
+  @clients << client
+  client
+end

data/features/transaction_fail.feature

@@ -0,0 +1,7 @@
+Feature: failure of transaction
+
+
+  Scenario: updating stale data
+    Given I start dp_map_manager on port 31331
+    And client attempts to perform transaction that updates stale value
+    Then client should be notified that transaction has failed