archipelago 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,7 +23,7 @@ require 'pp'
23
23
  require 'set'
24
24
  require 'archipelago/hashish'
25
25
  require 'archipelago/tranny'
26
- require 'archipelago/disco'
26
+ require 'archipelago/pirate'
27
27
 
28
28
  module Archipelago
29
29
 
@@ -51,6 +51,16 @@ module Archipelago
51
51
  end
52
52
  end
53
53
 
54
+ #
55
+ # Raised when a Dubloon tries to connect to this Archipelago::Treasure::Chest while
56
+ # we know that we are not the proper destination.
57
+ #
58
+ class WrongChestException < RuntimeError
59
+ def initialize(chest_id, predecessor_id, key)
60
+ super("#{key.inspect} is not between #{predecessor_id} and #{chest_id}, and should not be here")
61
+ end
62
+ end
63
+
54
64
  #
55
65
  # Raised whenever a method is called on an object that we dont know about.
56
66
  #
@@ -162,7 +172,7 @@ module Archipelago
162
172
  end
163
173
  #
164
174
  # Load some instance variables and replace @chest if we know that
165
- # it actually is not correct.
175
+ # it is incorrect.
166
176
  #
167
177
  def self._load(s)
168
178
  key, chest, transaction, chest_id = Marshal.load(s)
@@ -189,7 +199,7 @@ module Archipelago
189
199
  end
190
200
  #
191
201
  # This Dubloon will always have the same object_id, based on
192
- # @chest_id and @key and possibly @transaction.
202
+ # @chest_id, @key and possibly @transaction.
193
203
  #
194
204
  def object_id
195
205
  id = "#{@chest_id}:#{@key}"
@@ -211,20 +221,58 @@ module Archipelago
211
221
  def method_missing(meth, *args, &block)
212
222
  begin
213
223
  return @chest.call_instance_method(@key, meth, @transaction, *args, &block)
224
+ rescue WrongChestException => e
225
+ #
226
+ # We just have to find the new chest we belong at now...
227
+ #
228
+ new_chest_record = nil
229
+ if defined?(Archipelago::Pirate::BLACKBEARD)
230
+ Archipelago::Pirate::BLACKBEARD.update_services!
231
+ new_chest_record = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key)
232
+ new_chest_record = nil unless new_chest_record[:service].include?(@key, @transaction)
233
+ end
234
+ raise e unless new_chest_record
235
+
236
+ chest = new_chest_record
237
+
238
+ retry
214
239
  rescue DRb::DRbConnError => e
240
+ #
241
+ # Here is some fancy rescuing being done.
242
+ #
243
+ # First: If we have an MC we will try to reconnect to our actual chest.
244
+ #
245
+ # Second: If that failed, we will try to reconnect to the chest that (according to the BLACKBEARD)
246
+ # is responsible for our key - if it actually HAS our key.
247
+ #
248
+ # If this fails, lets raise hell.
249
+ #
250
+ new_chest_record = nil
215
251
  if defined?(Archipelago::Disco::MC)
216
- possible_replacements = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({:service_id => @chest_id}))
217
- raise e if possible_replacements.empty?
218
-
219
- @chest = possible_replacements[@chest_id][:service]
220
- CHEST_BY_SERVICE_ID[@chest_id] = @chest
221
-
222
- retry
223
- else
224
- raise e
252
+ new_chest_record = Archipelago::Disco::MC.lookup(Archipelago::Disco::Query.new({
253
+ :service_id => @chest_id
254
+ }))[@chest_id]
255
+ end
256
+ if new_chest_record.nil? && defined?(Archipelago::Pirate::BLACKBEARD)
257
+ Archipelago::Pirate::BLACKBEARD.update_services!(:validate => true)
258
+ new_chest_record = Archipelago::Pirate::BLACKBEARD.responsible_chest(@key)
259
+ new_chest_record = nil unless new_chest_record[:service].include?(@key, @transaction)
225
260
  end
261
+ raise e unless new_chest_record
262
+
263
+ chest = new_chest_record
264
+
265
+ retry
226
266
  end
227
267
  end
268
+
269
+ private
270
+
271
+ def chest=(record)
272
+ @chest = record[:service]
273
+ @chest_id = record[:service_id]
274
+ CHEST_BY_SERVICE_ID[@chest_id] = @chest
275
+ end
228
276
 
229
277
  end
230
278
 
@@ -245,19 +293,17 @@ module Archipelago
245
293
  # Initialize a Chest
246
294
  #
247
295
  # Will use a BerkeleyHashishProvider using treasure_chest.db in the same dir to get its hashes
248
- # if not <i>:persistence_provider</i> is given.
296
+ # if not <i>:persistence_directory</i> is given.
249
297
  #
250
298
  # Will try to recover crashed transaction every <i>:transaction_recovery_interval</i> seconds
251
299
  # or TRANSACTION_RECOVERY_INTERVAL if none is given.
252
300
  #
301
+ # Will store the actual data in a remote Archipelago::Dump network if <i>:officer</i> is given
302
+ # or in a local hash if not.
303
+ #
253
304
  # Will use Archipelago::Disco::Publishable by calling <b>initialize_publishable</b> with +options+.
254
305
  #
255
306
  def initialize(options = {})
256
- #
257
- # The provider of happy magic persistent hashes of different kinds.
258
- #
259
- @persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("treasure_chest.db"))
260
-
261
307
  #
262
308
  # Use the given options to initialize the publishable
263
309
  # instance variables.
@@ -290,11 +336,39 @@ module Archipelago
290
336
  # The magical persistent map that defines how we actually
291
337
  # store our data.
292
338
  #
293
- @db = @persistence_provider.get_cached_hashish("db")
339
+ if options[:officer]
340
+ @db = @persistence_provider.get_cached_hashish(:officer => options[:officer])
341
+ else
342
+ @db = @persistence_provider.get_cached_hashish(:name => "db")
343
+ end
294
344
 
295
345
  initialize_prepared(options[:transaction_recovery_interval] || TRANSACTION_RECOVERY_INTERVAL)
296
346
 
297
347
  CHEST_BY_SERVICE_ID[self.service_id] = self
348
+
349
+ #
350
+ # The provider of Chest index and other magic.
351
+ #
352
+ @captain = options[:captain] || (defined?(Archipelago::Pirate::BLACKBEARD) ? Archipelago::Pirate::BLACKBEARD : Archipelago::Pirate::Captain.new)
353
+
354
+ #
355
+ # Our predecessor. We will NOT accept remote calls for keys before this one.
356
+ #
357
+ @predecessor_id = nil
358
+ end
359
+
360
+ #
361
+ # Our predecessor is making itself public,
362
+ # so make sure we dont know about any keys we shouldnt.
363
+ #
364
+ def notify!(predecessor_id)
365
+ @predecessor_id = predecessor_id
366
+ while (to_check = @db.first) && !between?(to_check.first, @predecessor_id, service_id)
367
+ @db.forget(to_check.first)
368
+ end
369
+ while (to_check = @db.last) && !between?(to_check.first, @predecessor_id, service_id)
370
+ @db.forget(to_check.first)
371
+ end
298
372
  end
299
373
 
300
374
  #
@@ -434,6 +508,8 @@ module Archipelago
434
508
  # will be wrapped within the block sent to <b>with_transaction(transaction, &block)</b>.
435
509
  #
436
510
  def call_instance_method(key, method, transaction, *arguments, &block)
511
+ assert_mine(key)
512
+
437
513
  if transaction
438
514
  return call_with_transaction(key, method, transaction, *arguments, &block)
439
515
  else
@@ -562,10 +638,12 @@ module Archipelago
562
638
  # Copy each key and value from our private space to the real space
563
639
  #
564
640
  snapshot.each do |key, value|
565
- if value == :deleted
566
- @db.delete(key)
567
- else
568
- @db[key] = value
641
+ @db.without_lock do
642
+ if value == :deleted
643
+ @db.delete(key)
644
+ else
645
+ @db[key] = value
646
+ end
569
647
  end
570
648
  end
571
649
 
@@ -579,6 +657,62 @@ module Archipelago
579
657
 
580
658
  private
581
659
 
660
+ #
661
+ # Asserts that +key+ (as far as I know) belongs here.
662
+ #
663
+ # Raises WrongChestException if not.
664
+ #
665
+ def assert_mine(key)
666
+ unless between?(key, @predecessor_id, service_id)
667
+ raise WrongChestException.new(service_id, @predecessor_id, key)
668
+ end
669
+ end
670
+
671
+ #
672
+ # We don't want to publish until we have notified our successor of our id,
673
+ # and we need to repeat that when chests are lost or found.
674
+ #
675
+ def around_publish(&block)
676
+ notify_successor!
677
+ @jockey.subscribe(:lost, @captain.service_descriptions[:chests], service_id) do |record|
678
+ notify_successor!
679
+ end
680
+ @jockey.subscribe(:found, @captain.service_descriptions[:chests], service_id) do |record|
681
+ notify_successor!
682
+ end
683
+ yield
684
+ end
685
+
686
+ #
687
+ # We want to unsubscribe from the lost and found channel.
688
+ #
689
+ def around_unpublish(&block)
690
+ @jockey.unsubscribe(:lost, @captain.service_descriptions[:chests], service_id)
691
+ @jockey.unsubscribe(:found, @captain.service_descriptions[:chests], service_id)
692
+ yield
693
+ end
694
+
695
+ #
696
+ # We want to notify our successor when stuff happens.
697
+ #
698
+ def notify_successor!
699
+ successor = (@captain.successor(service_id) || {:service => self})[:service]
700
+ successor.notify!(service_id)
701
+ end
702
+
703
+ #
704
+ # Returns whether key is between id1 and id2 on the big service circle.
705
+ #
706
+ def between?(key, id1, id2)
707
+ if id1 == id2
708
+ return true
709
+ elsif id1 < id2
710
+ return key < id2 && key >= id1
711
+ elsif id1 > id2
712
+ return key < id2 || key >= id1
713
+ end
714
+ end
715
+
582
716
  #
583
717
  # Evaluates all data we have been told to evaluate in earlier runs.
584
718
  #
@@ -656,6 +790,7 @@ module Archipelago
656
790
  instance.with_transaction(transaction) do
657
791
  return_value = instance.send(method, *arguments, &block)
658
792
  end
793
+ return_value = Dubloon.new(key, self, transaction, service_id) if return_value == instance
659
794
  return return_value
660
795
  else
661
796
  return instance.send(method, *arguments, &block)
@@ -678,10 +813,12 @@ module Archipelago
678
813
  def call_without_transaction(key, method, *arguments, &block)
679
814
  instance = @db[key]
680
815
 
681
- raise UnknownObjectException(self, key) unless instance
816
+ raise UnknownObjectException.new(self, key) unless instance
682
817
 
683
818
  begin
684
- return instance.send(method, *arguments, &block)
819
+ return_value = instance.send(method, *arguments, &block)
820
+ return_value = Dubloon.new(key, self, nil, service_id) if return_value == instance
821
+ return return_value
685
822
  ensure
686
823
  @db.store_if_changed(key)
687
824
  end
@@ -704,10 +841,13 @@ module Archipelago
704
841
  #
705
842
  @snapshot_by_transaction_db = @persistence_provider.get_hashish("prepared")
706
843
  @snapshot_by_transaction_db.each do |serialized_transaction, serialized_snapshot|
707
- transaction = Marshal.load(transaction)
844
+ transaction = Marshal.load(serialized_transaction)
708
845
 
709
846
  @crashed << transaction
710
847
  @snapshot_by_transaction[transaction] = Marshal.load(serialized_snapshot)
848
+ @snapshot_by_transaction[transaction].each do |key, value|
849
+ @db.lock_on(key)
850
+ end
711
851
  end
712
852
  start_recovery_thread(transaction_recovery_interval)
713
853
  end
@@ -750,7 +890,13 @@ module Archipelago
750
890
  #
751
891
  def set(key, value, transaction)
752
892
  join!(transaction)
753
- value.assert_transaction(transaction) if Dubloon === value
893
+ if Dubloon === value
894
+ value.assert_transaction(transaction)
895
+ #
896
+ # Return the value if the value is in fact a proxy to what we are trying overwrite.
897
+ #
898
+ return value if value.object_id == Dubloon.new(key, self, transaction, self.service_id).object_id
899
+ end
754
900
 
755
901
  if transaction
756
902
  snapshot = @snapshot_by_transaction[transaction]
@@ -1,3 +1,3 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- exec 'irb', '-r', File.join(File.dirname(__FILE__), 'pirate.rb'), "-I", 'lib'
3
+ exec 'irb', '-r', File.join(File.dirname(__FILE__), 'pirate.rb'), '-r', File.join(File.dirname(__FILE__), 'officer.rb'), "-I", 'lib'
data/script/officer.rb ADDED
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'archipelago/sanitation'
4
+
5
+ begin
6
+ DRb.uri
7
+ rescue DRb::DRbServerNotFound
8
+ DRb.start_service(ENV["DRBURI"])
9
+ end
10
+ @o = Archipelago::Sanitation::Officer.new
data/script/pirate.rb CHANGED
@@ -2,6 +2,10 @@
2
2
 
3
3
  require 'archipelago/pirate'
4
4
 
5
- DRb.start_service
5
+ begin
6
+ DRb.uri
7
+ rescue DRb::DRbServerNotFound
8
+ DRb.start_service(ENV["DRBURI"])
9
+ end
6
10
  @p = Archipelago::Pirate::Captain.new
7
11
  @p.evaluate!(File.join(File.dirname(__FILE__), 'overloads.rb'))
data/script/services.rb CHANGED
@@ -9,7 +9,7 @@ $: << File.join(File.dirname(__FILE__), "..", "lib")
9
9
 
10
10
  require 'archipelago/treasure'
11
11
  require 'archipelago/tranny'
12
- require 'archipelago/cove'
12
+ require 'archipelago/dump'
13
13
 
14
14
  if ARGV.size > 1
15
15
  DRb.start_service(ARGV[1])
@@ -19,12 +19,19 @@ if ARGV.size > 1
19
19
  else
20
20
  DRb.start_service
21
21
  end
22
+ puts "using URI #{DRb.uri}"
22
23
 
23
- t = Archipelago::Tranny::Manager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tranny"))))
24
+ t = Archipelago::Tranny::Manager.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "tranny")))
24
25
  t.publish!
25
- c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "chest"))))
26
+ puts "published #{t.class} with id #{t.service_id}"
27
+
28
+ d = Archipelago::Dump::Site.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "dump")))
29
+ d.publish!
30
+ puts "published #{d.class} with id #{d.service_id}"
31
+
32
+ c = Archipelago::Treasure::Chest.new(:persistence_directory => Pathname.new(File.join(ARGV[0], "chest")),
33
+ :officer => Archipelago::Sanitation::CLEANER)
26
34
  c.publish!
27
- tank = Archipelago::Cove::Tanker.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tanker"))))
28
- tank.publish!
35
+ puts "published #{c.class} with id #{c.service_id}"
29
36
 
30
37
  DRb.thread.join
@@ -8,8 +8,8 @@ class DiscoBenchmark < Test::Unit::TestCase
8
8
  dj2 = TestJockey.new
9
9
  x = 0
10
10
  bm("Jockey#publish/lookup", :n => 10) do
11
- dj1.publish(Archipelago::Disco::Record.new({:service_id => x, :validator => Archipelago::Disco::MockValidator.new}))
12
- assert(!dj2.lookup(Archipelago::Disco::Query.new({:service_id => x})).empty?)
11
+ dj1.publish(Archipelago::Disco::Record.new({:service_id => x.to_s, :validator => Archipelago::Disco::MockValidator.new}))
12
+ assert(!dj2.lookup(Archipelago::Disco::Query.new({:service_id => x.to_s})).empty?)
13
13
  x += 1
14
14
  end
15
15
  dj1.publish(Archipelago::Disco::Record.new({:service_id => "brappa", :validator => Archipelago::Disco::MockValidator.new}))
data/tests/disco_test.rb CHANGED
@@ -15,7 +15,6 @@ end
15
15
  class DiscoTest < Test::Unit::TestCase
16
16
 
17
17
  def setup
18
- DRb.start_service
19
18
  @d1 = TestJockey.new(:thrifty_publishing => false,
20
19
  :thrifty_replying => false,
21
20
  :thrifty_caching => false)
@@ -47,7 +46,8 @@ class DiscoTest < Test::Unit::TestCase
47
46
  @lt = Thread.new do
48
47
  loop do
49
48
  begin
50
- @ltq << @listener.recv(1024)
49
+ d = @listener.recv(1024)
50
+ @ltq << d unless d =~ /UnPublish/
51
51
  rescue Exception => e
52
52
  puts e
53
53
  pp e.backtrace
@@ -59,31 +59,63 @@ class DiscoTest < Test::Unit::TestCase
59
59
  def teardown
60
60
  @d1.stop!
61
61
  @d2.stop!
62
- DRb.stop_service
63
62
  @lt.kill
64
63
  @listener.close
65
64
  end
66
65
 
67
66
  def test_publish_lookup
67
+ found_it = false
68
+ found_wrong = false
69
+
70
+ @d2.subscribe(:found, Archipelago::Disco::Query.new(:epa => "blar2"), 1) do
71
+ found_it = true
72
+ end
73
+ @d2.subscribe(:found, Archipelago::Disco::Query.new(:epa => "blar2x"), 1) do
74
+ found_wrong = true
75
+ end
76
+
68
77
  empty = true
69
78
  Thread.new do
70
79
  empty = @d2.lookup(Archipelago::Disco::Query.new(:epa => "blar2")).empty?
71
80
  end
72
81
 
73
82
  assert(empty)
74
-
75
- @d1.publish(Archipelago::Disco::Record.new(:service_id => 1,
83
+
84
+ @d1.publish(Archipelago::Disco::Record.new(:service_id => 100,
76
85
  :validator => Archipelago::Disco::MockValidator.new,
77
86
  :epa => "blar2"))
78
87
 
79
88
  assert_within(0.5) do
80
89
  !empty
81
90
  end
91
+
92
+ assert(found_it)
93
+ assert(!found_wrong)
82
94
  end
83
95
 
84
96
  def test_publish_invalidate
85
97
  @v1.valid = false
86
- assert(@d2.lookup(Archipelago::Disco::Query.new(:epa => "blar"), 0).empty?)
98
+
99
+ lost_it = false
100
+ lost_wrong = false
101
+
102
+ @d2.subscribe(:lost, Archipelago::Disco::Query.new(:epa => "blar"), 1) do
103
+ lost_it = true
104
+ end
105
+ @d2.subscribe(:lost, Archipelago::Disco::Query.new(:epa => "blarx"), 1) do
106
+ lost_wrong = true
107
+ end
108
+ @d2.subscribe(:lost, Archipelago::Disco::Query.new(:epa => "blarx"), 2) do
109
+ lost_wrong = true
110
+ end
111
+ @d2.unsubscribe(:lost, Archipelago::Disco::Query.new(:epa => "blarx"), 2)
112
+
113
+ assert(@d2.lookup(Archipelago::Disco::Query.new(:epa => "blar"), 0).validate!.empty?)
114
+
115
+ @d2.validate!
116
+
117
+ assert(lost_it)
118
+ assert(!lost_wrong)
87
119
  end
88
120
 
89
121
  def test_thrifty_publishing
@@ -112,7 +144,7 @@ class DiscoTest < Test::Unit::TestCase
112
144
  :validator => Archipelago::Disco::MockValidator.new,
113
145
  :service_id => 33))
114
146
  sleep 0.1
115
- assert(@ltq.empty?)
147
+ assert(@ltq.empty?, "we got messages while we shouldnt: #{@ltq.inspect}")
116
148
 
117
149
  c1 = Archipelago::Disco::Jockey.new(:thrifty_publishing => true)
118
150
  assert(!c1.lookup(Archipelago::Disco::Query.new(:glad => "ja")).empty?)