dp_stm_map 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. data/.gitignore +18 -0
  2. data/.rspec +1 -0
  3. data/Gemfile +4 -0
  4. data/LICENSE.txt +622 -0
  5. data/README.md +115 -0
  6. data/Rakefile +1 -0
  7. data/bin/dp_map_manager.rb +47 -0
  8. data/cucumber.yml +2 -0
  9. data/dp_stm_map.gemspec +29 -0
  10. data/features/client_reconnect.feature +13 -0
  11. data/features/persistence.feature +12 -0
  12. data/features/replication.feature +9 -0
  13. data/features/running_manager.feature +19 -0
  14. data/features/step_definitions/client_reconnect_steps.rb +28 -0
  15. data/features/step_definitions/persistence_steps.rb +49 -0
  16. data/features/step_definitions/replication_steps.rb +15 -0
  17. data/features/step_definitions/running_server_steps.rb +10 -0
  18. data/features/step_definitions/transaction_fail_steps.rb +11 -0
  19. data/features/support/env.rb +80 -0
  20. data/features/transaction_fail.feature +7 -0
  21. data/lib/dp_stm_map/Client.rb +268 -0
  22. data/lib/dp_stm_map/ClientLocalStore.rb +119 -0
  23. data/lib/dp_stm_map/InMemoryStmMap.rb +147 -0
  24. data/lib/dp_stm_map/Manager.rb +370 -0
  25. data/lib/dp_stm_map/Message.rb +126 -0
  26. data/lib/dp_stm_map/ObjectStore.rb +99 -0
  27. data/lib/dp_stm_map/version.rb +16 -0
  28. data/lib/dp_stm_map.rb +20 -0
  29. data/server.profile +547 -0
  30. data/spec/dp_stm_map/ClientLocalStore_spec.rb +78 -0
  31. data/spec/dp_stm_map/Client_spec.rb +133 -0
  32. data/spec/dp_stm_map/InMemoryStmMap_spec.rb +10 -0
  33. data/spec/dp_stm_map/Manager_spec.rb +323 -0
  34. data/spec/dp_stm_map/Message_spec.rb +21 -0
  35. data/spec/dp_stm_map/ObjectStore_spec.rb +87 -0
  36. data/spec/dp_stm_map/StmMap_shared.rb +432 -0
  37. metadata +235 -0
@@ -0,0 +1,268 @@
1
+ require 'socket'
2
+ require 'dp_stm_map'
3
+ require 'securerandom'
4
+ require 'digest/sha2'
5
+
6
+
7
+ module DpStmMap
8
+
9
+ class ShutdownError < StandardError
10
+ end
11
+
12
+
13
+
14
+ class DistributedPersistentStmMap
15
+
16
+ def initialize host, port, local_storage
17
+ @host=host
18
+ @port=port
19
+ @connect_listeners=[]
20
+ @disconnect_listeners=[]
21
+ @connect_state=:disconnected
22
+ @mutex=Mutex.new
23
+
24
+
25
+ @content={}
26
+
27
+
28
+ @state=ClientLocalStore.new local_storage
29
+
30
+
31
+ @validators=[]
32
+ @listeners=[]
33
+
34
+ @outgoing_queue=Queue.new
35
+
36
+
37
+ @outcome_futures={}
38
+
39
+ @transaction_id_condition_variable=ConditionVariable.new
40
+
41
+ end
42
+
43
+ def start
44
+
45
+
46
+ latch=Queue.new
47
+
48
+ @reading_thread=Thread.new do
49
+ begin
50
+ # puts "connecting"
51
+ @client_socket=nil
52
+ @client_socket=TCPSocket.new @host,@port
53
+
54
+ @client_socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
55
+ # puts "connected"
56
+ @connect_state=:connected
57
+
58
+
59
+ Thread.new do
60
+ begin
61
+ loop do
62
+ message=@outgoing_queue.pop
63
+ serialized=message.serialize
64
+ @client_socket.write([serialized.bytesize].pack("Q>"))
65
+ @client_socket.write(serialized)
66
+ @client_socket.flush
67
+ # puts "sent #{message}"
68
+ end
69
+ rescue => e
70
+ # puts "Error during processing: #{$!}"
71
+ # puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
72
+ end
73
+ end
74
+
75
+ send_to_server ClientHelloMessage.new(@state.current_transaction_sequence)
76
+
77
+
78
+ latch << "connected"
79
+
80
+ @connect_listeners.each do |listener|
81
+ listener.call
82
+ end
83
+
84
+
85
+
86
+ loop do
87
+ read=@client_socket.read(8)
88
+ # break unless read
89
+ len=read.unpack("Q>")[0]
90
+
91
+ msg=JsonMessage.deserialize(@client_socket.read(len))
92
+ # puts "got from server %s " % msg
93
+ if ClientTransactionOutcomeMessage === msg
94
+ # pp msg
95
+ @mutex.synchronize do
96
+ if @outcome_futures.has_key? msg.transaction_id
97
+ @outcome_futures.delete(msg.transaction_id).push msg
98
+ end
99
+ end
100
+ end
101
+ if TransactionMessage === msg
102
+ # pp msg
103
+ @mutex.synchronize do
104
+
105
+ changes=@state.update msg.transaction_sequence, msg.new_content, msg.transitions, msg.delete_content
106
+
107
+
108
+ @listeners.each do |listener|
109
+ begin
110
+ listener.call changes
111
+ rescue
112
+ end
113
+ end
114
+
115
+ @transaction_id_condition_variable.broadcast
116
+
117
+ end
118
+
119
+ end
120
+ end
121
+ rescue ShutdownError => e
122
+ # puts "shutdown"
123
+ rescue => e
124
+ # puts "Error during processing: #{$!}"
125
+ # puts "Backtrace:\n\t#{e.backtrace.join("\n\t")}"
126
+ # puts "error %s" % e
127
+ if @client_socket
128
+ @client_socket.close
129
+ end
130
+ if @connect_state == :connected
131
+ @connect_state=:disconnected
132
+ @disconnect_listeners.each do |listener|
133
+ listener.call
134
+ end
135
+ end
136
+ # puts "Exception: %s" % e
137
+ sleep 0.1
138
+ retry
139
+ ensure
140
+ if @connect_state == :connected
141
+ @connect_state=:disconnected
142
+ @disconnect_listeners.each do |listener|
143
+ listener.call
144
+ end
145
+ end
146
+ if @client_socket && !@client_socket.closed?
147
+ @client_socket.close
148
+ end
149
+ end
150
+
151
+ end
152
+
153
+ latch.pop
154
+ end
155
+
156
+ def stop
157
+ @reading_thread.raise ShutdownError
158
+ begin
159
+ @reading_thread.join
160
+ rescue => e
161
+ end
162
+ # puts "stopped"
163
+ end
164
+
165
+
166
+ def on_connected &block
167
+ @connect_listeners << block
168
+ end
169
+
170
+ def on_disconnected &block
171
+ @disconnect_listeners << block
172
+ end
173
+
174
+
175
+
176
+ def on_atomic &block
177
+ @listeners << block
178
+ end
179
+
180
+
181
+ def validate_atomic &block
182
+ @validators << block
183
+ end
184
+
185
+
186
+ def atomic timeout=nil
187
+
188
+ outcome_future=Queue.new
189
+
190
+
191
+ result=nil
192
+
193
+
194
+ tx_id=SecureRandom.uuid
195
+
196
+ view=AtomicView.new @state
197
+
198
+ @mutex.synchronize do
199
+ result=yield view
200
+ @outcome_futures[tx_id]=outcome_future
201
+ end
202
+
203
+ changes=view.changes
204
+
205
+ @validators.each do |validator|
206
+ validator.call changes
207
+ end
208
+
209
+
210
+
211
+ changes=view.changes
212
+
213
+ transitions={}
214
+
215
+ new_content={}
216
+
217
+ changes.each do |k,(old,new)|
218
+ transitions[k] = [content_digest(old), content_digest(new)]
219
+ new_content[content_digest(new)]=new
220
+ end
221
+
222
+ send_to_server ClientTransactionMessage.new tx_id, transitions, new_content
223
+
224
+ outcome=outcome_future.pop
225
+
226
+
227
+ if ClientTransactionSuccessfulMessage === outcome
228
+ @mutex.synchronize do
229
+ while @state.current_transaction_sequence < outcome.transaction_sequence
230
+ @transaction_id_condition_variable.wait(@mutex)
231
+ end
232
+ end
233
+ end
234
+
235
+
236
+ result
237
+
238
+ end
239
+
240
+
241
+ def content_digest content
242
+ unless content == nil
243
+ Digest::SHA2.hexdigest(content)
244
+ else
245
+ nil
246
+ end
247
+ end
248
+
249
+ def atomic_read
250
+ @mutex.synchronize do
251
+ yield AtomicReadView.new @state
252
+ end
253
+ end
254
+
255
+
256
+
257
+ # private
258
+
259
+ def send_to_server message
260
+ # puts "about to send #{message}"
261
+ @outgoing_queue << message
262
+ end
263
+
264
+
265
+ end
266
+
267
+
268
+ end
@@ -0,0 +1,119 @@
1
+ # dp_stm_map - Distributed and Persistent Software Transaction Map
2
+ # Copyright (C) 2013 Dragan Milic
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+
14
+ require 'threadsafe-lru'
15
+
16
+ module DpStmMap
17
+ class ClientLocalStore
18
+ def initialize store_dir
19
+
20
+ @store_dir=store_dir
21
+
22
+ @mapping_dir="%s/mapping" % store_dir
23
+ FileUtils.mkdir_p(@mapping_dir) unless File.exist? @mapping_dir
24
+
25
+ @content_dir="%s/content" % store_dir
26
+ FileUtils.mkdir_p(@content_dir) unless File.exist? @content_dir
27
+
28
+
29
+ transaction_sequence_file_name="%s/transaction_sequence.txt" % @store_dir
30
+
31
+ File.open(transaction_sequence_file_name,"w") {|f| f.write(0)} unless File.exist? transaction_sequence_file_name
32
+
33
+ @content_cache=ThreadSafeLru::LruCache.new 9000
34
+ @mapping_cache=ThreadSafeLru::LruCache.new 9000
35
+
36
+ end
37
+
38
+ def update transaction_sequence, new_content, updates, to_delete
39
+
40
+
41
+
42
+ File.open("%s/transaction_sequence.txt" % @store_dir,"w") {|f| f.write(transaction_sequence)}
43
+
44
+
45
+ new_content.each_pair do |k,v|
46
+ File.open("%s/%s" % [@content_dir,k],"w") {|f| f.write(v)}
47
+ end
48
+
49
+
50
+ changes=updates.inject({}) do |c, (k, new_hash)|
51
+ c[k]=[self[k],get_content(new_hash)]
52
+ c
53
+ end
54
+
55
+ to_delete.each do |hash|
56
+ File.delete("%s/%s" % [@content_dir,hash])
57
+ end
58
+
59
+ updates.each_pair do |k,v|
60
+ @mapping_cache.drop(k)
61
+ if v == nil
62
+ File.delete("%s/%s" % [@mapping_dir,k])
63
+ else
64
+ raise StandardError, "unknown content #{v} for key #{k}" unless has_content?(v)
65
+ File.open("%s/%s" % [@mapping_dir,k],"w") {|f| f.write(v)}
66
+ end
67
+ end
68
+
69
+
70
+
71
+ changes
72
+
73
+ end
74
+
75
+ def current_transaction_sequence
76
+ File.open("%s/transaction_sequence.txt" % @store_dir,"r") {|f| f.read().to_i}
77
+ end
78
+
79
+
80
+ def has_content? hash
81
+ File.exist?("%s/%s" % [@content_dir,hash])
82
+ end
83
+
84
+ def get_content h
85
+ @content_cache.get(h) do |hash|
86
+ if hash == nil
87
+ nil
88
+ else
89
+ File.open("%s/%s" % [@content_dir,hash],"r") {|f| f.read }
90
+ end
91
+ end
92
+ end
93
+
94
+ def get_mapping k
95
+ @mapping_cache.get(k) do |key|
96
+ if key == nil
97
+ nil
98
+ else
99
+ if File.exist? "%s/%s" % [@mapping_dir,key]
100
+ File.open("%s/%s" % [@mapping_dir,key],"r") {|f| f.read }
101
+ else
102
+ nil
103
+ end
104
+ end
105
+ end
106
+ end
107
+
108
+
109
+
110
+ def [] key
111
+ get_content(get_mapping(key))
112
+ end
113
+
114
+ def has_key? key
115
+ File.exist?("%s/%s" % [@mapping_dir,key])
116
+ end
117
+
118
+ end
119
+ end
@@ -0,0 +1,147 @@
1
+ # dp_stm_map - Distributed and Persistent Software Transaction Map
2
+ # Copyright (C) 2013 Dragan Milic
3
+ #
4
+ # This program is free software: you can redistribute it and/or modify
5
+ # it under the terms of the GNU General Public License as published by
6
+ # the Free Software Foundation, either version 3 of the License, or
7
+ # (at your option) any later version.
8
+ #
9
+ # This program is distributed in the hope that it will be useful,
10
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
+ # GNU General Public License for more details.
13
+
14
+ require 'thread'
15
+ require 'set'
16
+
17
+ module DpStmMap
18
+
19
+ class AtomicView
20
+
21
+ def initialize global_state
22
+ @global_state=global_state
23
+ @local_state={}
24
+ @to_delete=Set.new
25
+ end
26
+
27
+ def []= key, value
28
+ if value == nil
29
+ delete key
30
+ else
31
+ @local_state[key]=value
32
+ end
33
+ end
34
+
35
+ def [] key
36
+ if @local_state.has_key? key
37
+ @local_state[key]
38
+ else
39
+ @local_state[key]=@global_state[key]
40
+ end
41
+ end
42
+
43
+ def has_key? key
44
+ if @local_state.has_key?(key)
45
+ @local_state[key] != @global_state[key]
46
+ else
47
+ (not @to_delete.include?(key)) && @global_state.has_key?(key)
48
+ end
49
+ end
50
+
51
+ def delete key
52
+ if @local_state.has_key? key
53
+ @local_state.delete key
54
+ else
55
+ @to_delete << key
56
+ end
57
+ end
58
+
59
+ def changes
60
+ changes={}
61
+ @local_state.each do |k,v|
62
+ changes[k]=[@global_state[k],v]
63
+ end
64
+ @to_delete.each do |k|
65
+ changes[k]=[@global_state[k],nil]
66
+ end
67
+ changes
68
+ end
69
+
70
+ def commit
71
+ @to_delete.each do |key|
72
+ @global_state.delete key
73
+ end
74
+ @global_state.merge!(@local_state)
75
+ end
76
+
77
+ end
78
+
79
+ class AtomicReadView
80
+ def initialize state
81
+ @state=state
82
+ end
83
+
84
+ def [] key
85
+ @state[key]
86
+ end
87
+
88
+ def has_key? key
89
+ @state.has_key? key
90
+ end
91
+ end
92
+
93
+
94
+ class InMemoryStmMap
95
+
96
+ def initialize
97
+ @mutex=Mutex.new
98
+ @state={}
99
+ @validators=[]
100
+ @listeners=[]
101
+ end
102
+
103
+
104
+ def on_atomic &block
105
+ @listeners << block
106
+ end
107
+
108
+
109
+ def validate_atomic &block
110
+ @validators << block
111
+ end
112
+
113
+
114
+ def atomic timeout=nil
115
+ @mutex.synchronize do
116
+ view=AtomicView.new @state
117
+ result=yield view
118
+
119
+ changes=view.changes
120
+
121
+ @validators.each do |validator|
122
+ validator.call changes
123
+ end
124
+
125
+ view.commit
126
+
127
+ @listeners.each do |listener|
128
+ begin
129
+ listener.call changes
130
+ rescue
131
+ end
132
+ end
133
+
134
+ result
135
+ end
136
+ end
137
+
138
+ def atomic_read
139
+ @mutex.synchronize do
140
+ yield AtomicReadView.new @state
141
+ end
142
+ end
143
+
144
+
145
+ end
146
+
147
+ end