archipelago 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README +1 -1
- data/TODO +8 -17
- data/lib/archipelago.rb +0 -12
- data/lib/archipelago/client.rb +154 -17
- data/lib/archipelago/current.rb +1 -1
- data/lib/archipelago/disco.rb +269 -74
- data/lib/archipelago/dump.rb +279 -0
- data/lib/archipelago/hashish.rb +264 -108
- data/lib/archipelago/pirate.rb +52 -43
- data/lib/archipelago/sanitation.rb +268 -0
- data/lib/archipelago/tranny.rb +2 -4
- data/lib/archipelago/treasure.rb +173 -27
- data/script/{console → console.rb} +1 -1
- data/script/officer.rb +10 -0
- data/script/pirate.rb +5 -1
- data/script/services.rb +12 -5
- data/tests/disco_benchmark.rb +2 -2
- data/tests/disco_test.rb +39 -7
- data/tests/dump_test.rb +71 -0
- data/tests/pirate_test.rb +74 -21
- data/tests/sanitation_benchmark.rb +50 -0
- data/tests/sanitation_test.rb +219 -0
- data/tests/test_helper.rb +15 -3
- data/tests/tranny_test.rb +0 -2
- data/tests/treasure_benchmark.rb +6 -3
- data/tests/treasure_test.rb +43 -7
- metadata +13 -7
- data/lib/archipelago/cove.rb +0 -68
- data/lib/archipelago/exxon.rb +0 -138
- data/lib/archipelago/oneline.rb +0 -641
data/lib/archipelago/treasure.rb
CHANGED
@@ -23,7 +23,7 @@ require 'pp'
|
|
23
23
|
require 'set'
|
24
24
|
require 'archipelago/hashish'
|
25
25
|
require 'archipelago/tranny'
|
26
|
-
require 'archipelago/
|
26
|
+
require 'archipelago/pirate'
|
27
27
|
|
28
28
|
module Archipelago
|
29
29
|
|
@@ -51,6 +51,16 @@ module Archipelago
|
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
|
+
#
|
55
|
+
# Raised when a Dubloon tries to connect to this Archipelago::Treasure::Chest while
|
56
|
+
# we know that we are not the proper destination.
|
57
|
+
#
|
58
|
+
class WrongChestException < RuntimeError
|
59
|
+
def initialize(chest_id, predecessor_id, key)
|
60
|
+
super("#{key.inspect} is not between #{predecessor_id} and #{chest_id}, and should not be here")
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
54
64
|
#
|
55
65
|
# Raised whenever a method is called on an object that we dont know about.
|
56
66
|
#
|
@@ -162,7 +172,7 @@ module Archipelago
|
|
162
172
|
end
|
163
173
|
#
|
164
174
|
# Load some instance variables and replace @chest if we know that
|
165
|
-
# it
|
175
|
+
# it is incorrect.
|
166
176
|
#
|
167
177
|
def self._load(s)
|
168
178
|
key, chest, transaction, chest_id = Marshal.load(s)
|
@@ -189,7 +199,7 @@ module Archipelago
|
|
189
199
|
end
|
190
200
|
#
|
191
201
|
# This Dubloon will always have the same object_id, based on
|
192
|
-
# @chest_id
|
202
|
+
# @chest_id, @key and possibly @transaction.
|
193
203
|
#
|
194
204
|
def object_id
|
195
205
|
id = "#{@chest_id}:#{@key}"
|
@@ -211,20 +221,58 @@ module Archipelago
|
|
211
221
|
def method_missing(meth, *args, &block)
|
212
222
|
begin
|
213
223
|
return @chest.call_instance_method(@key, meth, @transaction, *args, &block)
|
224
|
+
rescue WrongChestException => e
|
225
|
+
#
|
226
|
+
# We just have to find the new chest we belong at now...
|
227
|
+
#
|
228
|
+
new_chest_record = nil
|
229
|
+
if defined?(Archipelago::Pirate::BLACKBEARD)
|
230
|
+
Archipelago::Pirate::BLACKBEARD.update_services!
|
231
|
+
new_chest_record = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key)
|
232
|
+
new_chest_record = nil unless new_chest_record[:service].include?(@key, @transaction)
|
233
|
+
end
|
234
|
+
raise e unless new_chest_record
|
235
|
+
|
236
|
+
chest = new_chest_record
|
237
|
+
|
238
|
+
retry
|
214
239
|
rescue DRb::DRbConnError => e
|
240
|
+
#
|
241
|
+
# Here is some fancy rescuing being done.
|
242
|
+
#
|
243
|
+
# First: If we have an MC we will try to reconnect to our actual chest.
|
244
|
+
#
|
245
|
+
# Second: If that failed, we will try to reconnect to the chest that (according to the BLACKBEARD)
|
246
|
+
# is responsible for our key - if it actually HAS our key.
|
247
|
+
#
|
248
|
+
# If this fails, lets raise hell.
|
249
|
+
#
|
250
|
+
new_chest_record = nil
|
215
251
|
if defined?(Archipelago::Disco::MC)
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
raise e
|
252
|
+
new_chest_record = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({
|
253
|
+
:service_id => @chest_id
|
254
|
+
}))[@chest_id]
|
255
|
+
end
|
256
|
+
if new_chest_record.nil? && defined?(Archipelago::Pirate::BLACKBEARD)
|
257
|
+
Archipelago::Pirate::BLACKBEARD.update_services!(:validate => true)
|
258
|
+
new_chest_record = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key)
|
259
|
+
new_chest_record = nil unless new_chest_record[:service].include?(@key, @transaction)
|
225
260
|
end
|
261
|
+
raise e unless new_chest_record
|
262
|
+
|
263
|
+
chest = new_chest_record
|
264
|
+
|
265
|
+
retry
|
226
266
|
end
|
227
267
|
end
|
268
|
+
|
269
|
+
private
|
270
|
+
|
271
|
+
def chest=(record)
|
272
|
+
@chest = record[:service]
|
273
|
+
@chest_id = record[:service_id]
|
274
|
+
CHEST_BY_SERVICE_ID[@chest_id] = @chest
|
275
|
+
end
|
228
276
|
|
229
277
|
end
|
230
278
|
|
@@ -245,19 +293,17 @@ module Archipelago
|
|
245
293
|
# Initialize a Chest
|
246
294
|
#
|
247
295
|
# Will use a BerkeleyHashishProvider using treasure_chest.db in the same dir to get its hashes
|
248
|
-
# if not <i>:
|
296
|
+
# if not <i>:persistence_directory</i> is given.
|
249
297
|
#
|
250
298
|
# Will try to recover crashed transaction every <i>:transaction_recovery_interval</i> seconds
|
251
299
|
# or TRANSACTION_RECOVERY_INTERVAL if none is given.
|
252
300
|
#
|
301
|
+
# Will store the actual data in a remote Archipelago::Dump network if <i>:officer</i> is given
|
302
|
+
# or in a local hash if not.
|
303
|
+
#
|
253
304
|
# Will use Archipelago::Disco::Publishable by calling <b>initialize_publishable</b> with +options+.
|
254
305
|
#
|
255
306
|
def initialize(options = {})
|
256
|
-
#
|
257
|
-
# The provider of happy magic persistent hashes of different kinds.
|
258
|
-
#
|
259
|
-
@persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("treasure_chest.db"))
|
260
|
-
|
261
307
|
#
|
262
308
|
# Use the given options to initialize the publishable
|
263
309
|
# instance variables.
|
@@ -290,11 +336,39 @@ module Archipelago
|
|
290
336
|
# The magical persistent map that defines how we actually
|
291
337
|
# store our data.
|
292
338
|
#
|
293
|
-
|
339
|
+
if options[:officer]
|
340
|
+
@db = @persistence_provider.get_cached_hashish(:officer => options[:officer])
|
341
|
+
else
|
342
|
+
@db = @persistence_provider.get_cached_hashish(:name => "db")
|
343
|
+
end
|
294
344
|
|
295
345
|
initialize_prepared(options[:transaction_recovery_interval] || TRANSACTION_RECOVERY_INTERVAL)
|
296
346
|
|
297
347
|
CHEST_BY_SERVICE_ID[self.service_id] = self
|
348
|
+
|
349
|
+
#
|
350
|
+
# The provider of Chest index and other magic.
|
351
|
+
#
|
352
|
+
@captain = options[:captain] || (defined?(Archipelago::Pirate::BLACKBEARD) ? Archipelago::Pirate::BLACKBEARD : Archipelago::Pirate::Captain.new)
|
353
|
+
|
354
|
+
#
|
355
|
+
# Our predecessor. We will NOT accept remote calls for keys before this one.
|
356
|
+
#
|
357
|
+
@predecessor_id = nil
|
358
|
+
end
|
359
|
+
|
360
|
+
#
|
361
|
+
# Our predecessor is making itself public,
|
362
|
+
# so make sure we dont know about any keys we shouldnt.
|
363
|
+
#
|
364
|
+
def notify!(predecessor_id)
|
365
|
+
@predecessor_id = predecessor_id
|
366
|
+
while (to_check = @db.first) && !between?(to_check.first, @predecessor_id, service_id)
|
367
|
+
@db.forget(to_check.first)
|
368
|
+
end
|
369
|
+
while (to_check = @db.last) && !between?(to_check.first, @predecessor_id, service_id)
|
370
|
+
@db.forget(to_check.first)
|
371
|
+
end
|
298
372
|
end
|
299
373
|
|
300
374
|
#
|
@@ -434,6 +508,8 @@ module Archipelago
|
|
434
508
|
# will be wrapped within the block sent to <b>with_transaction(transaction, &block)</b>.
|
435
509
|
#
|
436
510
|
def call_instance_method(key, method, transaction, *arguments, &block)
|
511
|
+
assert_mine(key)
|
512
|
+
|
437
513
|
if transaction
|
438
514
|
return call_with_transaction(key, method, transaction, *arguments, &block)
|
439
515
|
else
|
@@ -562,10 +638,12 @@ module Archipelago
|
|
562
638
|
# Copy each key and value from our private space to the real space
|
563
639
|
#
|
564
640
|
snapshot.each do |key, value|
|
565
|
-
|
566
|
-
|
567
|
-
|
568
|
-
|
641
|
+
@db.without_lock do
|
642
|
+
if value == :deleted
|
643
|
+
@db.delete(key)
|
644
|
+
else
|
645
|
+
@db[key] = value
|
646
|
+
end
|
569
647
|
end
|
570
648
|
end
|
571
649
|
|
@@ -579,6 +657,62 @@ module Archipelago
|
|
579
657
|
|
580
658
|
private
|
581
659
|
|
660
|
+
#
|
661
|
+
# Asserts that +key+ (as far as I know) belongs here.
|
662
|
+
#
|
663
|
+
# Raises WrongChestException if not.
|
664
|
+
#
|
665
|
+
def assert_mine(key)
|
666
|
+
unless between?(key, @predecessor_id, service_id)
|
667
|
+
raise WrongChestException.new(service_id, @predecessor_id, key)
|
668
|
+
end
|
669
|
+
end
|
670
|
+
|
671
|
+
#
|
672
|
+
# We don't want to publish until we have notified our successor of our id,
|
673
|
+
# and we need to repeat that when chests are lost or found.
|
674
|
+
#
|
675
|
+
def around_publish(&block)
|
676
|
+
notify_successor!
|
677
|
+
@jockey.subscribe(:lost, @captain.service_descriptions[:chests], service_id) do |record|
|
678
|
+
notify_successor!
|
679
|
+
end
|
680
|
+
@jockey.subscribe(:found, @captain.service_descriptions[:chests], service_id) do |record|
|
681
|
+
notify_successor!
|
682
|
+
end
|
683
|
+
yield
|
684
|
+
end
|
685
|
+
|
686
|
+
#
|
687
|
+
# We want to unsubscribe from the lost and found channel.
|
688
|
+
#
|
689
|
+
def around_unpublish(&block)
|
690
|
+
@jockey.unsubscribe(:lost, @captain.service_descriptions[:chests], service_id)
|
691
|
+
@jockey.unsubscribe(:found, @captain.service_descriptions[:chests], service_id)
|
692
|
+
yield
|
693
|
+
end
|
694
|
+
|
695
|
+
#
|
696
|
+
# We want to notify our successor when stuff happens.
|
697
|
+
#
|
698
|
+
def notify_successor!
|
699
|
+
successor = (@captain.successor(service_id) || {:service => self})[:service]
|
700
|
+
successor.notify!(service_id)
|
701
|
+
end
|
702
|
+
|
703
|
+
#
|
704
|
+
# Returns whether key is between id1 and id2 on the big service circle.
|
705
|
+
#
|
706
|
+
def between?(key, id1, id2)
|
707
|
+
if id1 == id2
|
708
|
+
return true
|
709
|
+
elsif id1 < id2
|
710
|
+
return key < id2 && key >= id1
|
711
|
+
elsif id1 > id2
|
712
|
+
return key < id2 || key >= id1
|
713
|
+
end
|
714
|
+
end
|
715
|
+
|
582
716
|
#
|
583
717
|
# Evaluates all data we have been told to evaluate in earlier runs.
|
584
718
|
#
|
@@ -656,6 +790,7 @@ module Archipelago
|
|
656
790
|
instance.with_transaction(transaction) do
|
657
791
|
return_value = instance.send(method, *arguments, &block)
|
658
792
|
end
|
793
|
+
return_value = Dubloon.new(key, self, transaction, service_id) if return_value == instance
|
659
794
|
return return_value
|
660
795
|
else
|
661
796
|
return instance.send(method, *arguments, &block)
|
@@ -678,10 +813,12 @@ module Archipelago
|
|
678
813
|
def call_without_transaction(key, method, *arguments, &block)
|
679
814
|
instance = @db[key]
|
680
815
|
|
681
|
-
raise UnknownObjectException(self, key) unless instance
|
816
|
+
raise UnknownObjectException.new(self, key) unless instance
|
682
817
|
|
683
818
|
begin
|
684
|
-
|
819
|
+
return_value = instance.send(method, *arguments, &block)
|
820
|
+
return_value = Dubloon.new(key, self, nil, service_id) if return_value == instance
|
821
|
+
return return_value
|
685
822
|
ensure
|
686
823
|
@db.store_if_changed(key)
|
687
824
|
end
|
@@ -704,10 +841,13 @@ module Archipelago
|
|
704
841
|
#
|
705
842
|
@snapshot_by_transaction_db = @persistence_provider.get_hashish("prepared")
|
706
843
|
@snapshot_by_transaction_db.each do |serialized_transaction, serialized_snapshot|
|
707
|
-
transaction = Marshal.load(
|
844
|
+
transaction = Marshal.load(serialized_transaction)
|
708
845
|
|
709
846
|
@crashed << transaction
|
710
847
|
@snapshot_by_transaction[transaction] = Marshal.load(serialized_snapshot)
|
848
|
+
@snapshot_by_transaction[transaction].each do |key, value|
|
849
|
+
@db.lock_on(key)
|
850
|
+
end
|
711
851
|
end
|
712
852
|
start_recovery_thread(transaction_recovery_interval)
|
713
853
|
end
|
@@ -750,7 +890,13 @@ module Archipelago
|
|
750
890
|
#
|
751
891
|
def set(key, value, transaction)
|
752
892
|
join!(transaction)
|
753
|
-
|
893
|
+
if Dubloon === value
|
894
|
+
value.assert_transaction(transaction)
|
895
|
+
#
|
896
|
+
# Return the value if the value is in fact a proxy to what we are trying overwrite.
|
897
|
+
#
|
898
|
+
return value if value.object_id == Dubloon.new(key, self, transaction, self.service_id).object_id
|
899
|
+
end
|
754
900
|
|
755
901
|
if transaction
|
756
902
|
snapshot = @snapshot_by_transaction[transaction]
|
data/script/officer.rb
ADDED
data/script/pirate.rb
CHANGED
data/script/services.rb
CHANGED
@@ -9,7 +9,7 @@ $: << File.join(File.dirname(__FILE__), "..", "lib")
|
|
9
9
|
|
10
10
|
require 'archipelago/treasure'
|
11
11
|
require 'archipelago/tranny'
|
12
|
-
require 'archipelago/
|
12
|
+
require 'archipelago/dump'
|
13
13
|
|
14
14
|
if ARGV.size > 1
|
15
15
|
DRb.start_service(ARGV[1])
|
@@ -19,12 +19,19 @@ if ARGV.size > 1
|
|
19
19
|
else
|
20
20
|
DRb.start_service
|
21
21
|
end
|
22
|
+
puts "using URI #{DRb.uri}"
|
22
23
|
|
23
|
-
t = Archipelago::Tranny::Manager.new(:
|
24
|
+
t = Archipelago::Tranny::Manager.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "tranny")))
|
24
25
|
t.publish!
|
25
|
-
|
26
|
+
puts "published #{t.class} with id #{t.service_id}"
|
27
|
+
|
28
|
+
d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "dump")))
|
29
|
+
d.publish!
|
30
|
+
puts "published #{d.class} with id #{d.service_id}"
|
31
|
+
|
32
|
+
c = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "chest")),
|
33
|
+
:officer => Archipelago::Sanitation::CLEANER)
|
26
34
|
c.publish!
|
27
|
-
|
28
|
-
tank.publish!
|
35
|
+
puts "published #{c.class} with id #{c.service_id}"
|
29
36
|
|
30
37
|
DRb.thread.join
|
data/tests/disco_benchmark.rb
CHANGED
@@ -8,8 +8,8 @@ class DiscoBenchmark < Test::Unit::TestCase
|
|
8
8
|
dj2 = TestJockey.new
|
9
9
|
x = 0
|
10
10
|
bm("Jockey#publish/lookup", :n => 10) do
|
11
|
-
dj1.publish(Archipelago::Disco::Record.new({:service_id => x, :validator => Archipelago::Disco::MockValidator.new}))
|
12
|
-
assert(!dj2.lookup(Archipelago::Disco::Query.new({:service_id => x})).empty?)
|
11
|
+
dj1.publish(Archipelago::Disco::Record.new({:service_id => x.to_s, :validator => Archipelago::Disco::MockValidator.new}))
|
12
|
+
assert(!dj2.lookup(Archipelago::Disco::Query.new({:service_id => x.to_s})).empty?)
|
13
13
|
x += 1
|
14
14
|
end
|
15
15
|
dj1.publish(Archipelago::Disco::Record.new({:service_id => "brappa", :validator => Archipelago::Disco::MockValidator.new}))
|
data/tests/disco_test.rb
CHANGED
@@ -15,7 +15,6 @@ end
|
|
15
15
|
class DiscoTest < Test::Unit::TestCase
|
16
16
|
|
17
17
|
def setup
|
18
|
-
DRb.start_service
|
19
18
|
@d1 = TestJockey.new(:thrifty_publishing => false,
|
20
19
|
:thrifty_replying => false,
|
21
20
|
:thrifty_caching => false)
|
@@ -47,7 +46,8 @@ class DiscoTest < Test::Unit::TestCase
|
|
47
46
|
@lt = Thread.new do
|
48
47
|
loop do
|
49
48
|
begin
|
50
|
-
|
49
|
+
d = @listener.recv(1024)
|
50
|
+
@ltq << d unless d =~ /UnPublish/
|
51
51
|
rescue Exception => e
|
52
52
|
puts e
|
53
53
|
pp e.backtrace
|
@@ -59,31 +59,63 @@ class DiscoTest < Test::Unit::TestCase
|
|
59
59
|
def teardown
|
60
60
|
@d1.stop!
|
61
61
|
@d2.stop!
|
62
|
-
DRb.stop_service
|
63
62
|
@lt.kill
|
64
63
|
@listener.close
|
65
64
|
end
|
66
65
|
|
67
66
|
def test_publish_lookup
|
67
|
+
found_it = false
|
68
|
+
found_wrong = false
|
69
|
+
|
70
|
+
@d2.subscribe(:found, Archipelago::Disco::Query.new(:epa => "blar2"), 1) do
|
71
|
+
found_it = true
|
72
|
+
end
|
73
|
+
@d2.subscribe(:found, Archipelago::Disco::Query.new(:epa => "blar2x"), 1) do
|
74
|
+
found_wrong = true
|
75
|
+
end
|
76
|
+
|
68
77
|
empty = true
|
69
78
|
Thread.new do
|
70
79
|
empty = @d2.lookup(Archipelago::Disco::Query.new(:epa => "blar2")).empty?
|
71
80
|
end
|
72
81
|
|
73
82
|
assert(empty)
|
74
|
-
|
75
|
-
@d1.publish(Archipelago::Disco::Record.new(:service_id =>
|
83
|
+
|
84
|
+
@d1.publish(Archipelago::Disco::Record.new(:service_id => 100,
|
76
85
|
:validator => Archipelago::Disco::MockValidator.new,
|
77
86
|
:epa => "blar2"))
|
78
87
|
|
79
88
|
assert_within(0.5) do
|
80
89
|
!empty
|
81
90
|
end
|
91
|
+
|
92
|
+
assert(found_it)
|
93
|
+
assert(!found_wrong)
|
82
94
|
end
|
83
95
|
|
84
96
|
def test_publish_invalidate
|
85
97
|
@v1.valid = false
|
86
|
-
|
98
|
+
|
99
|
+
lost_it = false
|
100
|
+
lost_wrong = false
|
101
|
+
|
102
|
+
@d2.subscribe(:lost, Archipelago::Disco::Query.new(:epa => "blar"), 1) do
|
103
|
+
lost_it = true
|
104
|
+
end
|
105
|
+
@d2.subscribe(:lost, Archipelago::Disco::Query.new(:epa => "blarx"), 1) do
|
106
|
+
lost_wrong = true
|
107
|
+
end
|
108
|
+
@d2.subscribe(:lost, Archipelago::Disco::Query.new(:epa => "blarx"), 2) do
|
109
|
+
lost_wrong = true
|
110
|
+
end
|
111
|
+
@d2.unsubscribe(:lost, Archipelago::Disco::Query.new(:epa => "blarx"), 2)
|
112
|
+
|
113
|
+
assert(@d2.lookup(Archipelago::Disco::Query.new(:epa => "blar"), 0).validate!.empty?)
|
114
|
+
|
115
|
+
@d2.validate!
|
116
|
+
|
117
|
+
assert(lost_it)
|
118
|
+
assert(!lost_wrong)
|
87
119
|
end
|
88
120
|
|
89
121
|
def test_thrifty_publishing
|
@@ -112,7 +144,7 @@ class DiscoTest < Test::Unit::TestCase
|
|
112
144
|
:validator => Archipelago::Disco::MockValidator.new,
|
113
145
|
:service_id => 33))
|
114
146
|
sleep 0.1
|
115
|
-
assert(@ltq.empty
|
147
|
+
assert(@ltq.empty?, "we got messages while we shouldnt: #{@ltq.inspect}")
|
116
148
|
|
117
149
|
c1 = Archipelago::Disco::Jockey.new(:thrifty_publishing => true)
|
118
150
|
assert(!c1.lookup(Archipelago::Disco::Query.new(:glad => "ja")).empty?)
|