dp_stm_map 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +18 -0
- data/.rspec +1 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +622 -0
- data/README.md +115 -0
- data/Rakefile +1 -0
- data/bin/dp_map_manager.rb +47 -0
- data/cucumber.yml +2 -0
- data/dp_stm_map.gemspec +29 -0
- data/features/client_reconnect.feature +13 -0
- data/features/persistence.feature +12 -0
- data/features/replication.feature +9 -0
- data/features/running_manager.feature +19 -0
- data/features/step_definitions/client_reconnect_steps.rb +28 -0
- data/features/step_definitions/persistence_steps.rb +49 -0
- data/features/step_definitions/replication_steps.rb +15 -0
- data/features/step_definitions/running_server_steps.rb +10 -0
- data/features/step_definitions/transaction_fail_steps.rb +11 -0
- data/features/support/env.rb +80 -0
- data/features/transaction_fail.feature +7 -0
- data/lib/dp_stm_map/Client.rb +268 -0
- data/lib/dp_stm_map/ClientLocalStore.rb +119 -0
- data/lib/dp_stm_map/InMemoryStmMap.rb +147 -0
- data/lib/dp_stm_map/Manager.rb +370 -0
- data/lib/dp_stm_map/Message.rb +126 -0
- data/lib/dp_stm_map/ObjectStore.rb +99 -0
- data/lib/dp_stm_map/version.rb +16 -0
- data/lib/dp_stm_map.rb +20 -0
- data/server.profile +547 -0
- data/spec/dp_stm_map/ClientLocalStore_spec.rb +78 -0
- data/spec/dp_stm_map/Client_spec.rb +133 -0
- data/spec/dp_stm_map/InMemoryStmMap_spec.rb +10 -0
- data/spec/dp_stm_map/Manager_spec.rb +323 -0
- data/spec/dp_stm_map/Message_spec.rb +21 -0
- data/spec/dp_stm_map/ObjectStore_spec.rb +87 -0
- data/spec/dp_stm_map/StmMap_shared.rb +432 -0
- metadata +235 -0
@@ -0,0 +1,268 @@
|
|
1
|
+
require 'socket'
|
2
|
+
require 'dp_stm_map'
|
3
|
+
require 'securerandom'
|
4
|
+
require 'digest/sha2'
|
5
|
+
|
6
|
+
|
7
|
+
module DpStmMap
|
8
|
+
|
9
|
+
class ShutdownError < StandardError
|
10
|
+
end
|
11
|
+
|
12
|
+
|
13
|
+
|
14
|
+
class DistributedPersistentStmMap
|
15
|
+
|
16
|
+
def initialize host, port, local_storage
|
17
|
+
@host=host
|
18
|
+
@port=port
|
19
|
+
@connect_listeners=[]
|
20
|
+
@disconnect_listeners=[]
|
21
|
+
@connect_state=:disconnected
|
22
|
+
@mutex=Mutex.new
|
23
|
+
|
24
|
+
|
25
|
+
@content={}
|
26
|
+
|
27
|
+
|
28
|
+
@state=ClientLocalStore.new local_storage
|
29
|
+
|
30
|
+
|
31
|
+
@validators=[]
|
32
|
+
@listeners=[]
|
33
|
+
|
34
|
+
@outgoing_queue=Queue.new
|
35
|
+
|
36
|
+
|
37
|
+
@outcome_futures={}
|
38
|
+
|
39
|
+
@transaction_id_condition_variable=ConditionVariable.new
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
def start
|
44
|
+
|
45
|
+
|
46
|
+
latch=Queue.new
|
47
|
+
|
48
|
+
@reading_thread=Thread.new do
|
49
|
+
begin
|
50
|
+
# puts "connecting"
|
51
|
+
@client_socket=nil
|
52
|
+
@client_socket=TCPSocket.new @host,@port
|
53
|
+
|
54
|
+
@client_socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
|
55
|
+
# puts "connected"
|
56
|
+
@connect_state=:connected
|
57
|
+
|
58
|
+
|
59
|
+
Thread.new do
|
60
|
+
begin
|
61
|
+
loop do
|
62
|
+
message=@outgoing_queue.pop
|
63
|
+
serialized=message.serialize
|
64
|
+
@client_socket.write([serialized.bytesize].pack("Q>"))
|
65
|
+
@client_socket.write(serialized)
|
66
|
+
@client_socket.flush
|
67
|
+
# puts "sent #{message}"
|
68
|
+
end
|
69
|
+
rescue => e
|
70
|
+
# puts "Error during processing: #{$!}"
|
71
|
+
# puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
send_to_server ClientHelloMessage.new(@state.current_transaction_sequence)
|
76
|
+
|
77
|
+
|
78
|
+
latch << "connected"
|
79
|
+
|
80
|
+
@connect_listeners.each do |listener|
|
81
|
+
listener.call
|
82
|
+
end
|
83
|
+
|
84
|
+
|
85
|
+
|
86
|
+
loop do
|
87
|
+
read=@client_socket.read(8)
|
88
|
+
# break unless read
|
89
|
+
len=read.unpack("Q>")[0]
|
90
|
+
|
91
|
+
msg=JsonMessage.deserialize(@client_socket.read(len))
|
92
|
+
# puts "got from server %s " % msg
|
93
|
+
if ClientTransactionOutcomeMessage === msg
|
94
|
+
# pp msg
|
95
|
+
@mutex.synchronize do
|
96
|
+
if @outcome_futures.has_key? msg.transaction_id
|
97
|
+
@outcome_futures.delete(msg.transaction_id).push msg
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
if TransactionMessage === msg
|
102
|
+
# pp msg
|
103
|
+
@mutex.synchronize do
|
104
|
+
|
105
|
+
changes=@state.update msg.transaction_sequence, msg.new_content, msg.transitions, msg.delete_content
|
106
|
+
|
107
|
+
|
108
|
+
@listeners.each do |listener|
|
109
|
+
begin
|
110
|
+
listener.call changes
|
111
|
+
rescue
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
@transaction_id_condition_variable.broadcast
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
end
|
120
|
+
end
|
121
|
+
rescue ShutdownError => e
|
122
|
+
# puts "shutdown"
|
123
|
+
rescue => e
|
124
|
+
# puts "Error during processing: #{$!}"
|
125
|
+
# puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
|
126
|
+
# puts "error %s" % e
|
127
|
+
if @client_socket
|
128
|
+
@client_socket.close
|
129
|
+
end
|
130
|
+
if @connect_state == :connected
|
131
|
+
@connect_state=:disconnected
|
132
|
+
@disconnect_listeners.each do |listener|
|
133
|
+
listener.call
|
134
|
+
end
|
135
|
+
end
|
136
|
+
# puts "Exception: %s" % e
|
137
|
+
sleep 0.1
|
138
|
+
retry
|
139
|
+
ensure
|
140
|
+
if @connect_state == :connected
|
141
|
+
@connect_state=:disconnected
|
142
|
+
@disconnect_listeners.each do |listener|
|
143
|
+
listener.call
|
144
|
+
end
|
145
|
+
end
|
146
|
+
if @client_socket && !@client_socket.closed?
|
147
|
+
@client_socket.close
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
end
|
152
|
+
|
153
|
+
latch.pop
|
154
|
+
end
|
155
|
+
|
156
|
+
def stop
|
157
|
+
@reading_thread.raise ShutdownError
|
158
|
+
begin
|
159
|
+
@reading_thread.join
|
160
|
+
rescue => e
|
161
|
+
end
|
162
|
+
# puts "stopped"
|
163
|
+
end
|
164
|
+
|
165
|
+
|
166
|
+
def on_connected &block
|
167
|
+
@connect_listeners << block
|
168
|
+
end
|
169
|
+
|
170
|
+
def on_disconnected &block
|
171
|
+
@disconnect_listeners << block
|
172
|
+
end
|
173
|
+
|
174
|
+
|
175
|
+
|
176
|
+
def on_atomic &block
|
177
|
+
@listeners << block
|
178
|
+
end
|
179
|
+
|
180
|
+
|
181
|
+
def validate_atomic &block
|
182
|
+
@validators << block
|
183
|
+
end
|
184
|
+
|
185
|
+
|
186
|
+
def atomic timeout=nil
|
187
|
+
|
188
|
+
outcome_future=Queue.new
|
189
|
+
|
190
|
+
|
191
|
+
result=nil
|
192
|
+
|
193
|
+
|
194
|
+
tx_id=SecureRandom.uuid
|
195
|
+
|
196
|
+
view=AtomicView.new @state
|
197
|
+
|
198
|
+
@mutex.synchronize do
|
199
|
+
result=yield view
|
200
|
+
@outcome_futures[tx_id]=outcome_future
|
201
|
+
end
|
202
|
+
|
203
|
+
changes=view.changes
|
204
|
+
|
205
|
+
@validators.each do |validator|
|
206
|
+
validator.call changes
|
207
|
+
end
|
208
|
+
|
209
|
+
|
210
|
+
|
211
|
+
changes=view.changes
|
212
|
+
|
213
|
+
transitions={}
|
214
|
+
|
215
|
+
new_content={}
|
216
|
+
|
217
|
+
changes.each do |k,(old,new)|
|
218
|
+
transitions[k] = [content_digest(old), content_digest(new)]
|
219
|
+
new_content[content_digest(new)]=new
|
220
|
+
end
|
221
|
+
|
222
|
+
send_to_server ClientTransactionMessage.new tx_id, transitions, new_content
|
223
|
+
|
224
|
+
outcome=outcome_future.pop
|
225
|
+
|
226
|
+
|
227
|
+
if ClientTransactionSuccessfulMessage === outcome
|
228
|
+
@mutex.synchronize do
|
229
|
+
while @state.current_transaction_sequence < outcome.transaction_sequence
|
230
|
+
@transaction_id_condition_variable.wait(@mutex)
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
|
236
|
+
result
|
237
|
+
|
238
|
+
end
|
239
|
+
|
240
|
+
|
241
|
+
def content_digest content
|
242
|
+
unless content == nil
|
243
|
+
Digest::SHA2.hexdigest(content)
|
244
|
+
else
|
245
|
+
nil
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
249
|
+
def atomic_read
|
250
|
+
@mutex.synchronize do
|
251
|
+
yield AtomicReadView.new @state
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
|
256
|
+
|
257
|
+
# private
|
258
|
+
|
259
|
+
def send_to_server message
|
260
|
+
# puts "about to send #{message}"
|
261
|
+
@outgoing_queue << message
|
262
|
+
end
|
263
|
+
|
264
|
+
|
265
|
+
end
|
266
|
+
|
267
|
+
|
268
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
# dp_stm_map - Distributed and Persistent Software Transaction Map
|
2
|
+
# Copyright (C) 2013 Dragan Milic
|
3
|
+
#
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
7
|
+
# (at your option) any later version.
|
8
|
+
#
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12
|
+
# GNU General Public License for more details.
|
13
|
+
|
14
|
+
require 'threadsafe-lru'
|
15
|
+
|
16
|
+
module DpStmMap
|
17
|
+
class ClientLocalStore
|
18
|
+
def initialize store_dir
|
19
|
+
|
20
|
+
@store_dir=store_dir
|
21
|
+
|
22
|
+
@mapping_dir="%s/mapping" % store_dir
|
23
|
+
FileUtils.mkdir_p(@mapping_dir) unless File.exist? @mapping_dir
|
24
|
+
|
25
|
+
@content_dir="%s/content" % store_dir
|
26
|
+
FileUtils.mkdir_p(@content_dir) unless File.exist? @content_dir
|
27
|
+
|
28
|
+
|
29
|
+
transaction_sequence_file_name="%s/transaction_sequence.txt" % @store_dir
|
30
|
+
|
31
|
+
File.open(transaction_sequence_file_name,"w") {|f| f.write(0)} unless File.exist? transaction_sequence_file_name
|
32
|
+
|
33
|
+
@content_cache=ThreadSafeLru::LruCache.new 9000
|
34
|
+
@mapping_cache=ThreadSafeLru::LruCache.new 9000
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
def update transaction_sequence, new_content, updates, to_delete
|
39
|
+
|
40
|
+
|
41
|
+
|
42
|
+
File.open("%s/transaction_sequence.txt" % @store_dir,"w") {|f| f.write(transaction_sequence)}
|
43
|
+
|
44
|
+
|
45
|
+
new_content.each_pair do |k,v|
|
46
|
+
File.open("%s/%s" % [@content_dir,k],"w") {|f| f.write(v)}
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
changes=updates.inject({}) do |c, (k, new_hash)|
|
51
|
+
c[k]=[self[k],get_content(new_hash)]
|
52
|
+
c
|
53
|
+
end
|
54
|
+
|
55
|
+
to_delete.each do |hash|
|
56
|
+
File.delete("%s/%s" % [@content_dir,hash])
|
57
|
+
end
|
58
|
+
|
59
|
+
updates.each_pair do |k,v|
|
60
|
+
@mapping_cache.drop(k)
|
61
|
+
if v == nil
|
62
|
+
File.delete("%s/%s" % [@mapping_dir,k])
|
63
|
+
else
|
64
|
+
raise StandardError, "unknown content #{v} for key #{k}" unless has_content?(v)
|
65
|
+
File.open("%s/%s" % [@mapping_dir,k],"w") {|f| f.write(v)}
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
|
71
|
+
changes
|
72
|
+
|
73
|
+
end
|
74
|
+
|
75
|
+
def current_transaction_sequence
|
76
|
+
File.open("%s/transaction_sequence.txt" % @store_dir,"r") {|f| f.read().to_i}
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
def has_content? hash
|
81
|
+
File.exist?("%s/%s" % [@content_dir,hash])
|
82
|
+
end
|
83
|
+
|
84
|
+
def get_content h
|
85
|
+
@content_cache.get(h) do |hash|
|
86
|
+
if hash == nil
|
87
|
+
nil
|
88
|
+
else
|
89
|
+
File.open("%s/%s" % [@content_dir,hash],"r") {|f| f.read }
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def get_mapping k
|
95
|
+
@mapping_cache.get(k) do |key|
|
96
|
+
if key == nil
|
97
|
+
nil
|
98
|
+
else
|
99
|
+
if File.exist? "%s/%s" % [@mapping_dir,key]
|
100
|
+
File.open("%s/%s" % [@mapping_dir,key],"r") {|f| f.read }
|
101
|
+
else
|
102
|
+
nil
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
|
109
|
+
|
110
|
+
def [] key
|
111
|
+
get_content(get_mapping(key))
|
112
|
+
end
|
113
|
+
|
114
|
+
def has_key? key
|
115
|
+
File.exist?("%s/%s" % [@mapping_dir,key])
|
116
|
+
end
|
117
|
+
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,147 @@
|
|
1
|
+
# dp_stm_map - Distributed and Persistent Software Transaction Map
|
2
|
+
# Copyright (C) 2013 Dragan Milic
|
3
|
+
#
|
4
|
+
# This program is free software: you can redistribute it and/or modify
|
5
|
+
# it under the terms of the GNU General Public License as published by
|
6
|
+
# the Free Software Foundation, either version 3 of the License, or
|
7
|
+
# (at your option) any later version.
|
8
|
+
#
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12
|
+
# GNU General Public License for more details.
|
13
|
+
|
14
|
+
require 'thread'
|
15
|
+
require 'set'
|
16
|
+
|
17
|
+
module DpStmMap
|
18
|
+
|
19
|
+
class AtomicView
|
20
|
+
|
21
|
+
def initialize global_state
|
22
|
+
@global_state=global_state
|
23
|
+
@local_state={}
|
24
|
+
@to_delete=Set.new
|
25
|
+
end
|
26
|
+
|
27
|
+
def []= key, value
|
28
|
+
if value == nil
|
29
|
+
delete key
|
30
|
+
else
|
31
|
+
@local_state[key]=value
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def [] key
|
36
|
+
if @local_state.has_key? key
|
37
|
+
@local_state[key]
|
38
|
+
else
|
39
|
+
@local_state[key]=@global_state[key]
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def has_key? key
|
44
|
+
if @local_state.has_key?(key)
|
45
|
+
@local_state[key] != @global_state[key]
|
46
|
+
else
|
47
|
+
(not @to_delete.include?(key)) && @global_state.has_key?(key)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def delete key
|
52
|
+
if @local_state.has_key? key
|
53
|
+
@local_state.delete key
|
54
|
+
else
|
55
|
+
@to_delete << key
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def changes
|
60
|
+
changes={}
|
61
|
+
@local_state.each do |k,v|
|
62
|
+
changes[k]=[@global_state[k],v]
|
63
|
+
end
|
64
|
+
@to_delete.each do |k|
|
65
|
+
changes[k]=[@global_state[k],nil]
|
66
|
+
end
|
67
|
+
changes
|
68
|
+
end
|
69
|
+
|
70
|
+
def commit
|
71
|
+
@to_delete.each do |key|
|
72
|
+
@global_state.delete key
|
73
|
+
end
|
74
|
+
@global_state.merge!(@local_state)
|
75
|
+
end
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
class AtomicReadView
|
80
|
+
def initialize state
|
81
|
+
@state=state
|
82
|
+
end
|
83
|
+
|
84
|
+
def [] key
|
85
|
+
@state[key]
|
86
|
+
end
|
87
|
+
|
88
|
+
def has_key? key
|
89
|
+
@state.has_key? key
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
class InMemoryStmMap
|
95
|
+
|
96
|
+
def initialize
|
97
|
+
@mutex=Mutex.new
|
98
|
+
@state={}
|
99
|
+
@validators=[]
|
100
|
+
@listeners=[]
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
def on_atomic &block
|
105
|
+
@listeners << block
|
106
|
+
end
|
107
|
+
|
108
|
+
|
109
|
+
def validate_atomic &block
|
110
|
+
@validators << block
|
111
|
+
end
|
112
|
+
|
113
|
+
|
114
|
+
def atomic timeout=nil
|
115
|
+
@mutex.synchronize do
|
116
|
+
view=AtomicView.new @state
|
117
|
+
result=yield view
|
118
|
+
|
119
|
+
changes=view.changes
|
120
|
+
|
121
|
+
@validators.each do |validator|
|
122
|
+
validator.call changes
|
123
|
+
end
|
124
|
+
|
125
|
+
view.commit
|
126
|
+
|
127
|
+
@listeners.each do |listener|
|
128
|
+
begin
|
129
|
+
listener.call changes
|
130
|
+
rescue
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
result
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
def atomic_read
|
139
|
+
@mutex.synchronize do
|
140
|
+
yield AtomicReadView.new @state
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
|
145
|
+
end
|
146
|
+
|
147
|
+
end
|