archipelago 0.2.4 → 0.2.5
Sign up to get free protection for your applications and to get access to all the features.
- data/README +2 -1
- data/TODO +10 -0
- data/lib/archipelago/client.rb +99 -0
- data/lib/archipelago/cove.rb +68 -0
- data/lib/archipelago/current.rb +3 -0
- data/lib/archipelago/disco.rb +2 -9
- data/lib/archipelago/exxon.rb +138 -0
- data/lib/archipelago/hashish.rb +19 -6
- data/lib/archipelago/oneline.rb +641 -0
- data/lib/archipelago/pirate.rb +21 -56
- data/lib/archipelago.rb +6 -0
- data/script/services.rb +11 -5
- data/tests/test_helper.rb +1 -0
- metadata +6 -4
- data/script/chest.rb +0 -21
- data/script/tranny.rb +0 -21
data/README
CHANGED
@@ -4,6 +4,7 @@ It consists of several different parts, that can be used standalone or in conjun
|
|
4
4
|
|
5
5
|
== Dependencies:
|
6
6
|
Archipelago::Hashish::BerkeleyHashishProvider:: ruby bdb: http://moulon.inra.fr/ruby/bdb.html
|
7
|
+
String:: ruby inline: http://www.zenspider.com/ZSS/Products/RubyInline/
|
7
8
|
|
8
9
|
== Sub packages:
|
9
10
|
Archipelago::Disco:: A UDP multicast discovery service useful to find services in your network with a minimum of configuration.
|
@@ -31,7 +32,7 @@ To run a transaction manager and a database server, just do the following (from
|
|
31
32
|
|
32
33
|
:include:script/services.rb
|
33
34
|
|
34
|
-
To set up an Archipelago::Pirate::Captain do the following (from script/pirate.rb):
|
35
|
+
To set up an Archipelago::Pirate::Captain and load some debugging overloads do the following (from script/pirate.rb):
|
35
36
|
|
36
37
|
:include:script/pirate.rb
|
37
38
|
|
data/TODO
CHANGED
@@ -5,12 +5,22 @@
|
|
5
5
|
* Or: Create migration methods that move objects between Chests opon
|
6
6
|
startup and shutdown, and make them keep backups at each others
|
7
7
|
persistence backends.
|
8
|
+
* Or: Create something that stores data the way Chord does (with erasure
|
9
|
+
codes) but doesnt use the same look up mechanism.
|
10
|
+
* Problem: We still have to implement the entire maintenance protocol
|
11
|
+
of Chord (continously checking if our data is safely replicated across
|
12
|
+
the network, continously checking that our data belong with us)
|
13
|
+
|
14
|
+
* Replace Raider with some well known near-optimal erasure code, for example
|
15
|
+
Online Codes: http://en.wikipedia.org/wiki/Online_codes
|
8
16
|
|
9
17
|
* Make Chest aware about whether transactions have affected it 'for real' ie
|
10
18
|
check whether the instance before the call differs from the instance after
|
11
19
|
the call. Preferably without incurring performance lossage.
|
12
20
|
* For example: Make Dubloon proxy targets able to provide a dirty_state
|
13
21
|
method that decide whether they are to be considered dirty, clean or auto-select.
|
22
|
+
* This is now done. But it is not yet used to provide intelligence for the
|
23
|
+
transaction mechanism. How should it compare dirty state before and after?
|
14
24
|
|
15
25
|
* Test the transaction recovery mechanism of Chest.
|
16
26
|
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# Archipelago - a distributed computing toolkit for ruby
|
2
|
+
# Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
|
3
|
+
#
|
4
|
+
# This program is free software; you can redistribute it and/or
|
5
|
+
# modify it under the terms of the GNU General Public License
|
6
|
+
# as published by the Free Software Foundation; either version 2
|
7
|
+
# of the License, or (at your option) any later version.
|
8
|
+
#
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12
|
+
# GNU General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU General Public License
|
15
|
+
# along with this program; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
17
|
+
|
18
|
+
require 'archipelago/disco'
|
19
|
+
|
20
|
+
module Archipelago
|
21
|
+
|
22
|
+
module Client
|
23
|
+
|
24
|
+
INITIAL_SERVICE_UPDATE_INTERVAL = 1
|
25
|
+
MAXIMUM_SERVICE_UPDATE_INTERVAL = 60
|
26
|
+
|
27
|
+
class Base
|
28
|
+
attr_reader :jockey
|
29
|
+
#
|
30
|
+
# Initialize an instance using Archipelago::Disco::MC or <i>:jockey</i> if given,
|
31
|
+
# or a new Archipelago::Disco::Jockey if none, that looks for new services
|
32
|
+
# <i>:initial_service_update_interval</i> or INITIAL_SERVICE_UPDATE_INTERVAL,
|
33
|
+
# when it starts and never slower than every <i>:maximum_service_update_interval</i>
|
34
|
+
# or MAXIMUM_SERVICE_UPDATE_INTERVAL.
|
35
|
+
#
|
36
|
+
def initialize(options = {})
|
37
|
+
setup(options)
|
38
|
+
end
|
39
|
+
#
|
40
|
+
# Sets up this instance with the given +options+.
|
41
|
+
#
|
42
|
+
def setup(options = {})
|
43
|
+
@jockey.stop! if defined?(@jockey) && @jockey != Archipelago::Disco::MC
|
44
|
+
if defined?(Archipelago::Disco::MC)
|
45
|
+
@jockey = options[:jockey] || Archipelago::Disco::MC
|
46
|
+
else
|
47
|
+
@jockey = options[:jockey] || Archipelago::Disco::Jockey.new
|
48
|
+
end
|
49
|
+
|
50
|
+
@initial_service_update_interval = options[:initial_service_update_interval] || INITIAL_SERVICE_UPDATE_INTERVAL
|
51
|
+
@maximum_service_update_interval = options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL
|
52
|
+
end
|
53
|
+
|
54
|
+
#
|
55
|
+
# Stops the service update thread for this Pirate and also unpublishes and/or
|
56
|
+
# stops the Jockey.
|
57
|
+
#
|
58
|
+
def stop!
|
59
|
+
@service_update_thread.kill if @service_update_thread
|
60
|
+
unless defined?(Archipelago::Disco::MC) && @jockey && @jockey == Archipelago::Disco::MC
|
61
|
+
@jockey.stop!
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
#
|
66
|
+
# Override this to do whatever you want to when finding services.
|
67
|
+
#
|
68
|
+
def update_services!
|
69
|
+
# /moo
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
#
|
75
|
+
# Start a thread looking up existing chests between every
|
76
|
+
# +initial+ and +maximum+ seconds.
|
77
|
+
#
|
78
|
+
def start_service_updater
|
79
|
+
update_services!
|
80
|
+
@service_update_thread = Thread.start do
|
81
|
+
standoff = @initial_service_update_interval
|
82
|
+
loop do
|
83
|
+
begin
|
84
|
+
sleep(standoff)
|
85
|
+
standoff *= 2
|
86
|
+
standoff = @maximum_service_update_interval if standoff > @maximum_service_update_interval
|
87
|
+
update_services!
|
88
|
+
rescue Exception => e
|
89
|
+
puts e
|
90
|
+
pp e.backtrace
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# Archipelago - a distributed computing toolkit for ruby
|
2
|
+
# Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
|
3
|
+
#
|
4
|
+
# This program is free software; you can redistribute it and/or
|
5
|
+
# modify it under the terms of the GNU General Public License
|
6
|
+
# as published by the Free Software Foundation; either version 2
|
7
|
+
# of the License, or (at your option) any later version.
|
8
|
+
#
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12
|
+
# GNU General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU General Public License
|
15
|
+
# along with this program; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
17
|
+
|
18
|
+
require 'archipelago/disco'
|
19
|
+
require 'archipelago/hashish'
|
20
|
+
require 'archipelago/exxon'
|
21
|
+
require 'archipelago/raider'
|
22
|
+
|
23
|
+
module Archipelago
|
24
|
+
|
25
|
+
module Cove
|
26
|
+
|
27
|
+
class Tanker
|
28
|
+
|
29
|
+
#
|
30
|
+
# The Tanker can be published.
|
31
|
+
#
|
32
|
+
include Archipelago::Disco::Publishable
|
33
|
+
|
34
|
+
def initialize(options = {})
|
35
|
+
#
|
36
|
+
# The provider of happy magic persistent hashes of different kinds.
|
37
|
+
#
|
38
|
+
@persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("cove_tanker.db"))
|
39
|
+
|
40
|
+
@db = @persistence_provider.get_hashish("db")
|
41
|
+
|
42
|
+
@valdez = options[:valdez] || Archipelago::Exxon::HAZELWOOD
|
43
|
+
|
44
|
+
#
|
45
|
+
# Use the given options to initialize the publishable
|
46
|
+
# instance variables.
|
47
|
+
#
|
48
|
+
initialize_publishable(options)
|
49
|
+
|
50
|
+
end
|
51
|
+
|
52
|
+
def store_part(key, split_part)
|
53
|
+
@db[key] = Marshal.dump(split_part)
|
54
|
+
end
|
55
|
+
|
56
|
+
def get_part(key)
|
57
|
+
if @db.include?(key)
|
58
|
+
return Marshal.load(@db[key])
|
59
|
+
else
|
60
|
+
return nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
end
|
65
|
+
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
data/lib/archipelago/current.rb
CHANGED
data/lib/archipelago/disco.rb
CHANGED
@@ -22,6 +22,7 @@ require 'pp'
|
|
22
22
|
require 'archipelago/current'
|
23
23
|
require 'drb'
|
24
24
|
require 'set'
|
25
|
+
require 'digest/sha1'
|
25
26
|
|
26
27
|
module Archipelago
|
27
28
|
|
@@ -252,7 +253,7 @@ module Archipelago
|
|
252
253
|
#
|
253
254
|
def initialize(hash)
|
254
255
|
raise "Record must have a :service_id" unless hash.include?(:service_id)
|
255
|
-
raise "Record must have a :validator" unless hash.include?(:validator)
|
256
|
+
raise "Record must have a :validator" unless hash.include?(:validator) && hash[:validator].respond_to?(:valid?)
|
256
257
|
super(hash)
|
257
258
|
end
|
258
259
|
#
|
@@ -532,17 +533,9 @@ module Archipelago
|
|
532
533
|
recipient, data = @outgoing.pop
|
533
534
|
if recipient
|
534
535
|
address, port = recipient.split(/:/)
|
535
|
-
if $DEBUG
|
536
|
-
puts "sending to #{address}:#{port}:"
|
537
|
-
puts Marshal.dump(data)
|
538
|
-
end
|
539
536
|
@unisender.send(Marshal.dump(data), 0, address, port.to_i)
|
540
537
|
else
|
541
538
|
begin
|
542
|
-
if $DEBUG
|
543
|
-
puts "broadcasting to #{@multiaddress}:#{@multiport}:"
|
544
|
-
puts Marshal.dump(data)
|
545
|
-
end
|
546
539
|
@sender.write(Marshal.dump(data))
|
547
540
|
rescue Errno::ECONNREFUSED => e
|
548
541
|
retry
|
@@ -0,0 +1,138 @@
|
|
1
|
+
# Archipelago - a distributed computing toolkit for ruby
|
2
|
+
# Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
|
3
|
+
#
|
4
|
+
# This program is free software; you can redistribute it and/or
|
5
|
+
# modify it under the terms of the GNU General Public License
|
6
|
+
# as published by the Free Software Foundation; either version 2
|
7
|
+
# of the License, or (at your option) any later version.
|
8
|
+
#
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12
|
+
# GNU General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU General Public License
|
15
|
+
# along with this program; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
17
|
+
|
18
|
+
require 'archipelago/client'
|
19
|
+
|
20
|
+
module Archipelago
|
21
|
+
|
22
|
+
module Exxon
|
23
|
+
|
24
|
+
TANKER_DESCRIPTION = {
|
25
|
+
:class => 'Archipelago::Cove::Tanker'
|
26
|
+
}
|
27
|
+
|
28
|
+
PARTS = 3
|
29
|
+
|
30
|
+
#
|
31
|
+
# Raised when you try to do stuff without any remote database
|
32
|
+
# available.
|
33
|
+
#
|
34
|
+
class NoRemoteDatabaseAvailableException < RuntimeError
|
35
|
+
def initialize(valdez)
|
36
|
+
super("#{valdez} can not find any remote database for you")
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class Valdez < Archipelago::Client::Base
|
41
|
+
attr_reader :tankers
|
42
|
+
def initialize(options = {})
|
43
|
+
super(options)
|
44
|
+
|
45
|
+
start_service_updater
|
46
|
+
end
|
47
|
+
|
48
|
+
def setup(options = {})
|
49
|
+
super(options)
|
50
|
+
|
51
|
+
@parts = options[:parts] || PARTS
|
52
|
+
@tanker_description = TANKER_DESCRIPTION.merge(options[:tanker_description] || {})
|
53
|
+
end
|
54
|
+
|
55
|
+
def []=(key, value)
|
56
|
+
split_parts = Archipelago::Raider.split(value, @parts)
|
57
|
+
followers = responsible_tankers(key, split_parts.size)
|
58
|
+
0.upto(followers.size - 1) do |n|
|
59
|
+
followers[n].store_part(key, split_parts[n])
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def [](key)
|
64
|
+
return_value = nil
|
65
|
+
|
66
|
+
split_parts = []
|
67
|
+
followers = responsible_tankers(key, (1 << @parts) - 1)
|
68
|
+
while split_parts.size < @parts
|
69
|
+
split_parts << followers.shift.get_part(key)
|
70
|
+
end
|
71
|
+
begin
|
72
|
+
return_value = Archipelago::Raider.join(*split_parts.compact)
|
73
|
+
rescue Archipelago::Raider::NotEnoughBulkError => e
|
74
|
+
if followers.empty?
|
75
|
+
raise e
|
76
|
+
else
|
77
|
+
split_parts << followers.shift.get_part(key)
|
78
|
+
retry
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
return return_value
|
83
|
+
end
|
84
|
+
|
85
|
+
def update_services!
|
86
|
+
@tankers = @jockey.lookup(Archipelago::Disco::Query.new(@tanker_description), 0)
|
87
|
+
end
|
88
|
+
|
89
|
+
def responsible_tankers(key, n)
|
90
|
+
raise NoRemoteDatabaseAvailableException.new(self) if @tankers.empty?
|
91
|
+
|
92
|
+
responsible_ids = []
|
93
|
+
|
94
|
+
key_id = Digest::SHA1.hexdigest(Marshal.dump(key))
|
95
|
+
sorted_tanker_ids = @tankers.keys.sort
|
96
|
+
0.upto(sorted_tanker_ids.size - 1) do |index|
|
97
|
+
id = sorted_tanker_ids[index]
|
98
|
+
if id > key_id
|
99
|
+
responsible_ids = get_some(sorted_tanker_ids, index, n)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
responsible_ids = get_some(sorted_tanker_ids, 0, n)
|
103
|
+
|
104
|
+
return responsible_ids.collect do |id|
|
105
|
+
@tankers[id][:service]
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def responsible_tanker(key)
|
110
|
+
return responsible_tankers(key, 1).first
|
111
|
+
end
|
112
|
+
|
113
|
+
private
|
114
|
+
|
115
|
+
def get_some(array, index, n)
|
116
|
+
rval = []
|
117
|
+
while rval.size < n
|
118
|
+
index = 0 if index > array.size - 1
|
119
|
+
rval << array[index]
|
120
|
+
index += 1
|
121
|
+
end
|
122
|
+
return rval
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|
126
|
+
|
127
|
+
#
|
128
|
+
# The default Archipelago::Exxon::Valdez that is always available for lookups is Archipelago::Exxon::HAZELWOOD.
|
129
|
+
#
|
130
|
+
# If you really need to you can customize it by defining HAZELWOOD_OPTIONS before loading disco.rb, and if you REALLY
|
131
|
+
# need to you can disable it completely by setting HAZELWOOD_DISABLED to true.
|
132
|
+
#
|
133
|
+
HAZELWOOD = Valdez.new(defined?(HAZELWOOD_OPTIONS) ? HAZELWOOD_OPTIONS : {}) unless defined?(HAZELWOOD_DISABLED) && HAZELWOOD_DISABLED
|
134
|
+
|
135
|
+
|
136
|
+
end
|
137
|
+
|
138
|
+
end
|
data/lib/archipelago/hashish.rb
CHANGED
@@ -115,15 +115,28 @@ module Archipelago
|
|
115
115
|
#
|
116
116
|
def store_if_changed(key)
|
117
117
|
@lock.synchronize_on(key) do
|
118
|
-
|
119
|
-
serialized_key = Marshal.dump(key)
|
120
118
|
|
121
119
|
value = @content[key]
|
122
|
-
serialized_value = Marshal.dump(value)
|
123
|
-
old_serialized_value = @content_db[serialized_key]
|
124
120
|
|
125
|
-
|
126
|
-
|
121
|
+
if value.respond_to?(:dirty?)
|
122
|
+
|
123
|
+
if value.dirty?
|
124
|
+
serialized_value = Marshal.dump(value)
|
125
|
+
serialized_key = Marshal.dump(key)
|
126
|
+
write_to_db(key, serialized_key, serialized_value, value)
|
127
|
+
value.is_clean! if value.respond_to?(:is_clean!)
|
128
|
+
end
|
129
|
+
|
130
|
+
else
|
131
|
+
|
132
|
+
serialized_value = Marshal.dump(value)
|
133
|
+
serialized_key = Marshal.dump(key)
|
134
|
+
old_serialized_value = @content_db[serialized_key]
|
135
|
+
|
136
|
+
write_to_db(key, serialized_key, serialized_value, value) if old_serialized_value && old_serialized_value != serialized_value
|
137
|
+
|
138
|
+
end
|
139
|
+
|
127
140
|
end
|
128
141
|
end
|
129
142
|
#
|
@@ -0,0 +1,641 @@
|
|
1
|
+
# Archipelago - a distributed computing toolkit for ruby
|
2
|
+
# Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
|
3
|
+
#
|
4
|
+
# This program is free software; you can redistribute it and/or
|
5
|
+
# modify it under the terms of the GNU General Public License
|
6
|
+
# as published by the Free Software Foundation; either version 2
|
7
|
+
# of the License, or (at your option) any later version.
|
8
|
+
#
|
9
|
+
# This program is distributed in the hope that it will be useful,
|
10
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
11
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
12
|
+
# GNU General Public License for more details.
|
13
|
+
#
|
14
|
+
# You should have received a copy of the GNU General Public License
|
15
|
+
# along with this program; if not, write to the Free Software
|
16
|
+
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
|
17
|
+
|
18
|
+
require 'rubygems'
|
19
|
+
require 'inline'
|
20
|
+
require 'pp'
|
21
|
+
|
22
|
+
#
|
23
|
+
# Adds a few methods to String, some using the inline gem.
|
24
|
+
#
|
25
|
+
class String
|
26
|
+
|
27
|
+
E = [0.1, 0.01]
|
28
|
+
Q = [6, 3]
|
29
|
+
THRESHOLDS = [1024]
|
30
|
+
|
31
|
+
class << self
|
32
|
+
private
|
33
|
+
#
|
34
|
+
# Get an F value for given epsilon value
|
35
|
+
#
|
36
|
+
def get_f(e)
|
37
|
+
Math.log(e ** 2 / 4) / Math.log(1 - (e / 2))
|
38
|
+
end
|
39
|
+
#
|
40
|
+
# Get a p distribution with a given name and epsilon value.
|
41
|
+
#
|
42
|
+
# Will return a C variable definition for this distribution.
|
43
|
+
#
|
44
|
+
def _get_p_distribution(name, e)
|
45
|
+
f = get_f(e)
|
46
|
+
rval = "double #{name}[] = {0.0,"
|
47
|
+
p1 = 1 - ( (1 + (1 / f) ) / (1 + e) )
|
48
|
+
1.upto(f) do |n|
|
49
|
+
if n == 1
|
50
|
+
rval << "#{p1}"
|
51
|
+
else
|
52
|
+
rval << "#{( ( 1 - p1 ) * f ) / ( ( f - 1 ) * n * ( n - 1 ) )}"
|
53
|
+
end
|
54
|
+
rval << "," unless n + 1 > f
|
55
|
+
end
|
56
|
+
rval << "};\n"
|
57
|
+
return rval
|
58
|
+
end
|
59
|
+
#
|
60
|
+
# Get all the p distributions we want.
|
61
|
+
#
|
62
|
+
# Will return a C variable definition for this distribution.
|
63
|
+
#
|
64
|
+
def get_p_distribution
|
65
|
+
rval = ""
|
66
|
+
0.upto(E.size - 1) do |n|
|
67
|
+
rval << _get_p_distribution("P#{n}", E[n])
|
68
|
+
end
|
69
|
+
return rval
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
inline do |builder|
|
74
|
+
#
|
75
|
+
# We need math.h
|
76
|
+
#
|
77
|
+
builder.include("<math.h>")
|
78
|
+
#
|
79
|
+
# We need sys/time.h
|
80
|
+
#
|
81
|
+
builder.include("<sys/time.h>")
|
82
|
+
#
|
83
|
+
# Define the p distribution array so that it is easily accessible.
|
84
|
+
#
|
85
|
+
builder.prefix(get_p_distribution)
|
86
|
+
#
|
87
|
+
# Define a sane random number generator.
|
88
|
+
#
|
89
|
+
builder.prefix("#define RAND (((double)random())/((double)RAND_MAX+1))")
|
90
|
+
builder.prefix("#define RANDOM(n) (random() % (n))")
|
91
|
+
#
|
92
|
+
# Get a random degree.
|
93
|
+
#
|
94
|
+
# Will define as many get_degree{n} functions as we need.
|
95
|
+
#
|
96
|
+
degree_methods = ""
|
97
|
+
0.upto(E.size - 1) do |n|
|
98
|
+
degree_methods += <<EOC
|
99
|
+
static long
|
100
|
+
get_degree#{n}(size)
|
101
|
+
{
|
102
|
+
long rval = 1;
|
103
|
+
double d = RAND - P#{n}[rval];
|
104
|
+
while (d > 0.0) {
|
105
|
+
rval ++;
|
106
|
+
d = d - P#{n}[rval];
|
107
|
+
}
|
108
|
+
if (rval > size)
|
109
|
+
return size;
|
110
|
+
else
|
111
|
+
return rval;
|
112
|
+
}
|
113
|
+
EOC
|
114
|
+
end
|
115
|
+
builder.prefix(degree_methods)
|
116
|
+
#
|
117
|
+
# Get the function to produce random degrees for us.
|
118
|
+
#
|
119
|
+
# Will go through our THRESHOLDS and return the proper degree-method.
|
120
|
+
#
|
121
|
+
get_degree_func = <<EOC
|
122
|
+
long (*get_degree_func(long size))(long)
|
123
|
+
{
|
124
|
+
EOC
|
125
|
+
0.upto(THRESHOLDS.size - 1) do |n|
|
126
|
+
get_degree_func += <<EOC
|
127
|
+
if (size < #{THRESHOLDS[n]})
|
128
|
+
return &get_degree#{n};
|
129
|
+
EOC
|
130
|
+
end
|
131
|
+
get_degree_func += <<EOC
|
132
|
+
return &get_degree#{THRESHOLDS.size};
|
133
|
+
EOC
|
134
|
+
get_degree_func += <<EOC
|
135
|
+
}
|
136
|
+
EOC
|
137
|
+
builder.prefix(get_degree_func)
|
138
|
+
#
|
139
|
+
# Check if a certain block is known.
|
140
|
+
#
|
141
|
+
builder.prefix <<EOC
|
142
|
+
static char
|
143
|
+
get_is_known(VALUE decode_status, long block_number)
|
144
|
+
{
|
145
|
+
long byte_in_status = block_number / 8;
|
146
|
+
char bit_in_byte = 1 << (block_number % 8);
|
147
|
+
return ((RSTRING(decode_status)->ptr[byte_in_status]) & bit_in_byte) == bit_in_byte;
|
148
|
+
}
|
149
|
+
EOC
|
150
|
+
#
|
151
|
+
# Set that a certain block is known.
|
152
|
+
#
|
153
|
+
builder.prefix <<EOC
|
154
|
+
static void
|
155
|
+
set_is_known(VALUE decode_status, long block_number)
|
156
|
+
{
|
157
|
+
long byte_in_status = block_number / 8;
|
158
|
+
char bit_in_byte = 1 << (block_number % 8);
|
159
|
+
rb_str_modify(decode_status);
|
160
|
+
RSTRING(decode_status)->ptr[byte_in_status] = RSTRING(decode_status)->ptr[byte_in_status] | bit_in_byte;
|
161
|
+
}
|
162
|
+
EOC
|
163
|
+
#
|
164
|
+
# Returns whether this String is decoded enough.
|
165
|
+
#
|
166
|
+
# Will first go through our decode_status one byte a time
|
167
|
+
# and then check the relevant bits of the last byte.
|
168
|
+
#
|
169
|
+
builder.c <<EOC
|
170
|
+
static VALUE
|
171
|
+
_oneline_done(VALUE decode_status)
|
172
|
+
{
|
173
|
+
long tmp;
|
174
|
+
if (decode_status == Qnil)
|
175
|
+
return Qfalse;
|
176
|
+
for (tmp = 0; tmp < (RSTRING(decode_status)->len - 1); tmp++) {
|
177
|
+
if (( (unsigned char) RSTRING(decode_status)->ptr[tmp] ) != 255) {
|
178
|
+
return Qfalse;
|
179
|
+
}
|
180
|
+
}
|
181
|
+
for (tmp = 0; tmp < (RSTRING(self)->len % 8); tmp++) {
|
182
|
+
if (! ( (unsigned char) RSTRING(self)->ptr[tmp] ) & (1 << tmp)) {
|
183
|
+
return Qfalse;
|
184
|
+
}
|
185
|
+
}
|
186
|
+
return Qtrue;
|
187
|
+
}
|
188
|
+
EOC
|
189
|
+
#
|
190
|
+
# Set a data or aux block with a known value.
|
191
|
+
#
|
192
|
+
# Will both update ourselves or our aux_blocks depending on index,
|
193
|
+
# then set the decode_status so that this byte is officially known.
|
194
|
+
#
|
195
|
+
builder.prefix <<EOC
|
196
|
+
static void
|
197
|
+
set_value(VALUE aux_blocks, long index, char value, VALUE self, VALUE decode_status)
|
198
|
+
{
|
199
|
+
if (index < RSTRING(self)->len) {
|
200
|
+
rb_str_modify(self);
|
201
|
+
RSTRING(self)->ptr[index] = value;
|
202
|
+
set_is_known(decode_status, index);
|
203
|
+
} else {
|
204
|
+
rb_str_modify(aux_blocks);
|
205
|
+
RSTRING(aux_blocks)->ptr[index - RSTRING(self)->len] = value;
|
206
|
+
set_is_known(decode_status, index);
|
207
|
+
}
|
208
|
+
}
|
209
|
+
EOC
|
210
|
+
#
|
211
|
+
# Get a data or aux block.
|
212
|
+
#
|
213
|
+
# Gets from ourselves or our aux_blocks depending on the index.
|
214
|
+
#
|
215
|
+
builder.prefix <<EOC
|
216
|
+
static char
|
217
|
+
get_value(VALUE aux_blocks, long index, VALUE self)
|
218
|
+
{
|
219
|
+
if (index < RSTRING(self)->len) {
|
220
|
+
return RSTRING(self)->ptr[index];
|
221
|
+
} else {
|
222
|
+
return RSTRING(aux_blocks)->ptr[index - RSTRING(self)->len];
|
223
|
+
}
|
224
|
+
}
|
225
|
+
EOC
|
226
|
+
#
|
227
|
+
# Decode the given chunk until we dont get any new
|
228
|
+
# data from it.
|
229
|
+
#
|
230
|
+
builder.c <<EOC
|
231
|
+
static VALUE
|
232
|
+
_decode_chunk(VALUE chunk, VALUE decode_status, VALUE aux_blocks)
|
233
|
+
{
|
234
|
+
long seed = ( (long *) RSTRING(chunk)->ptr )[0];
|
235
|
+
long size = ( (long *) RSTRING(chunk)->ptr )[1];
|
236
|
+
long tmp, tmp2;
|
237
|
+
char got_new_block = 0;
|
238
|
+
long (*degree_func)() = get_degree_func(size);
|
239
|
+
|
240
|
+
// Go through the chunk.
|
241
|
+
for (tmp = 8; tmp < RSTRING(chunk)->len; tmp++) {
|
242
|
+
long degree;
|
243
|
+
|
244
|
+
// Seed with our seed and the index in the chunk.
|
245
|
+
srandom(seed + tmp);
|
246
|
+
|
247
|
+
//printf("looking at check block %d(%d):", tmp, RSTRING(chunk)->ptr[tmp]);
|
248
|
+
|
249
|
+
// Get the degree of this chunk.
|
250
|
+
degree = degree_func(size);
|
251
|
+
// If it is degree 1 then we can deduce the data/aux block immediately.
|
252
|
+
if (degree == 1) {
|
253
|
+
long block_nr;
|
254
|
+
|
255
|
+
// Seed with our seed, the chunk index and the degree.
|
256
|
+
srandom(seed + tmp + degree);
|
257
|
+
|
258
|
+
// This block can be either from ourselves or from our aux blocks.
|
259
|
+
block_nr = RANDOM(size + RSTRING(aux_blocks)->len);
|
260
|
+
|
261
|
+
//printf(" %d", block_nr);
|
262
|
+
|
263
|
+
// If it is not already known, then set it as known. Otherwise just log about its state.
|
264
|
+
if (!get_is_known(decode_status, block_nr)) {
|
265
|
+
set_value(aux_blocks, block_nr, RSTRING(chunk)->ptr[tmp], self, decode_status);
|
266
|
+
got_new_block = 1;
|
267
|
+
//printf(" => [%d]=%d", block_nr, RSTRING(chunk)->ptr[tmp]);
|
268
|
+
} else {
|
269
|
+
//printf(" [%d]=%d", block_nr, RSTRING(chunk)->ptr[tmp]);
|
270
|
+
}
|
271
|
+
} else {
|
272
|
+
// If it is NOT degree 1, then we go through all the blocks it is xored from
|
273
|
+
// and see if we have exactly one missing.
|
274
|
+
long missing_blocks;
|
275
|
+
long missing_block;
|
276
|
+
// Create the xor_sum with this block of the chunk.
|
277
|
+
char xor_sum = RSTRING(chunk)->ptr[tmp];
|
278
|
+
|
279
|
+
for (missing_blocks = tmp2 = degree; tmp2 > 0; tmp2--) {
|
280
|
+
long block_nr;
|
281
|
+
|
282
|
+
srandom(seed + tmp + tmp2);
|
283
|
+
|
284
|
+
block_nr = RANDOM(size + RSTRING(aux_blocks)->len);
|
285
|
+
|
286
|
+
//printf(" %d", block_nr);
|
287
|
+
|
288
|
+
// If the block we are looking at is known, then we decrement missing_blocks
|
289
|
+
// and continue with our xor_sum.
|
290
|
+
if (get_is_known(decode_status, block_nr)) {
|
291
|
+
missing_blocks--;
|
292
|
+
xor_sum = xor_sum ^ get_value(aux_blocks, block_nr, self);
|
293
|
+
//printf("!");
|
294
|
+
} else {
|
295
|
+
missing_block = block_nr;
|
296
|
+
//printf("?");
|
297
|
+
}
|
298
|
+
}
|
299
|
+
// If exactly one block is missing, set is as the final xor_sum.
|
300
|
+
if (missing_blocks == 1) {
|
301
|
+
set_value(aux_blocks, missing_block, xor_sum, self, decode_status);
|
302
|
+
got_new_block = 1;
|
303
|
+
//printf(" => [%d]=%d", missing_block, xor_sum);
|
304
|
+
}
|
305
|
+
}
|
306
|
+
//printf("\\n");
|
307
|
+
fflush(NULL);
|
308
|
+
}
|
309
|
+
|
310
|
+
return got_new_block ? Qtrue : Qfalse;
|
311
|
+
}
|
312
|
+
EOC
|
313
|
+
#
|
314
|
+
# Get a check block from a message.
|
315
|
+
#
|
316
|
+
builder.prefix <<EOC
|
317
|
+
static char
|
318
|
+
get_check_block(VALUE self, long seed, long index, VALUE aux_blocks, long (*degree_func)(long))
|
319
|
+
{
|
320
|
+
long degree;
|
321
|
+
int empty_block = 1;
|
322
|
+
char block;
|
323
|
+
char found_block;
|
324
|
+
|
325
|
+
// Seed with our seed and the index of this block in the chunk.
|
326
|
+
srandom(seed + index);
|
327
|
+
|
328
|
+
//printf("creating check block %d:", index);
|
329
|
+
|
330
|
+
// Find [degree] blocks.
|
331
|
+
for (degree = degree_func(RSTRING(self)->len); degree > 0; degree--) {
|
332
|
+
long block_number;
|
333
|
+
char found_block;
|
334
|
+
|
335
|
+
// Seed with our seed, the index in the chunk, and the degree.
|
336
|
+
srandom(seed + index + degree);
|
337
|
+
|
338
|
+
block_number = RANDOM(RSTRING(self)->len + RSTRING(aux_blocks)->len);
|
339
|
+
// Depending on whether the block is in our data or our own, debug about it
|
340
|
+
// and remember what it was.
|
341
|
+
if (block_number < RSTRING(self)->len) {
|
342
|
+
found_block = RSTRING(self)->ptr[block_number];
|
343
|
+
//printf(" d[%d]=%d", block_number, RSTRING(self)->ptr[block_number]);
|
344
|
+
} else {
|
345
|
+
found_block = RSTRING(aux_blocks)->ptr[block_number - RSTRING(self)->len];
|
346
|
+
//printf(" a[%d]=%d", block_number - RSTRING(self)->len, RSTRING(aux_blocks)->ptr[block_number - RSTRING(self)->len]);
|
347
|
+
}
|
348
|
+
// Create the block by xoring.
|
349
|
+
if (empty_block) {
|
350
|
+
block = found_block;
|
351
|
+
empty_block = 0;
|
352
|
+
} else {
|
353
|
+
block = block ^ found_block;
|
354
|
+
}
|
355
|
+
}
|
356
|
+
//printf(" => %d\\n", block);
|
357
|
+
return block;
|
358
|
+
}
|
359
|
+
EOC
|
360
|
+
#
|
361
|
+
# Fill our aux blocks with the proper data.
|
362
|
+
#
|
363
|
+
builder.c <<EOC
|
364
|
+
static void
|
365
|
+
_fill_aux_blocks(long size, long q)
|
366
|
+
{
|
367
|
+
char aux_blocks[size];
|
368
|
+
char initialized_aux_blocks[size];
|
369
|
+
long tmp, tmp2;
|
370
|
+
|
371
|
+
for (tmp = 0; tmp < size; tmp++)
|
372
|
+
initialized_aux_blocks[tmp] = 0;
|
373
|
+
|
374
|
+
srandom(RSTRING(self)->len);
|
375
|
+
|
376
|
+
// For each block of our data.
|
377
|
+
for (tmp = 0; tmp < RSTRING(self)->len; tmp++) {
|
378
|
+
//printf("creating aux blocks for d[%d](%d):", tmp, RSTRING(self)->ptr[tmp]);
|
379
|
+
// Create q blocks.
|
380
|
+
for (tmp2 = 0; tmp2 < q; tmp2++) {
|
381
|
+
long aux_block_nr = RANDOM(size);
|
382
|
+
//printf(" [%d]", aux_block_nr);
|
383
|
+
// If this aux block is already initialized, just xor to it, otherwise set it.
|
384
|
+
if (initialized_aux_blocks[aux_block_nr]) {
|
385
|
+
aux_blocks[aux_block_nr] = aux_blocks[aux_block_nr] ^ RSTRING(self)->ptr[tmp];
|
386
|
+
} else {
|
387
|
+
aux_blocks[aux_block_nr] = RSTRING(self)->ptr[tmp];
|
388
|
+
initialized_aux_blocks[aux_block_nr] = 1;
|
389
|
+
}
|
390
|
+
//printf(" => %d", aux_blocks[aux_block_nr]);
|
391
|
+
}
|
392
|
+
//printf("\\n");
|
393
|
+
}
|
394
|
+
rb_ivar_set(self, rb_intern("@_aux_blocks"), rb_str_new(aux_blocks, size));
|
395
|
+
}
|
396
|
+
EOC
|
397
|
+
#
|
398
|
+
# Returns a chunk of coded data for a given message.
|
399
|
+
#
|
400
|
+
builder.c <<EOC
|
401
|
+
static VALUE
|
402
|
+
_get_chunk(long chunk_size, VALUE aux_blocks)
|
403
|
+
{
|
404
|
+
unsigned long tmp;
|
405
|
+
long seed;
|
406
|
+
struct timeval tmp_timeval;
|
407
|
+
char chunk[chunk_size];
|
408
|
+
long (*degree_func)() = get_degree_func(RSTRING(self)->len);
|
409
|
+
|
410
|
+
// Seed with timeofday
|
411
|
+
gettimeofday(&tmp_timeval, NULL);
|
412
|
+
srandom(tmp_timeval.tv_usec);
|
413
|
+
// Get the seed for this chunk.
|
414
|
+
seed = random();
|
415
|
+
|
416
|
+
( (long *) chunk) [0] = seed;
|
417
|
+
( (long *) chunk) [1] = RSTRING(self)->len;
|
418
|
+
|
419
|
+
// Get the requested number of check blocks and append to this chunk.
|
420
|
+
for (tmp = 8; tmp < chunk_size; tmp++) {
|
421
|
+
chunk[tmp] = get_check_block(self, seed, tmp, aux_blocks, degree_func);
|
422
|
+
}
|
423
|
+
|
424
|
+
return rb_str_new(chunk, chunk_size);
|
425
|
+
}
|
426
|
+
EOC
|
427
|
+
#
|
428
|
+
# Create the @_aux_hash so that we know what data blocks each aux block represents.
|
429
|
+
#
|
430
|
+
builder.c <<EOC
|
431
|
+
static void
|
432
|
+
_fill_aux_hash(long q, long size)
|
433
|
+
{
|
434
|
+
long tmp, tmp2;
|
435
|
+
VALUE aux_ary = rb_ary_new();
|
436
|
+
VALUE set_func = rb_intern("[]=");
|
437
|
+
VALUE get_func = rb_intern("[]");
|
438
|
+
VALUE push_func = rb_intern("<<");
|
439
|
+
VALUE size_func = rb_intern("size");
|
440
|
+
|
441
|
+
srandom(RSTRING(self)->len);
|
442
|
+
|
443
|
+
// Go through all our blocks
|
444
|
+
for (tmp = 0; tmp < RSTRING(self)->len; tmp++) {
|
445
|
+
// And for each block q aux blocks.
|
446
|
+
for (tmp2 = 0; tmp2 < q; tmp2++) {
|
447
|
+
// And build an Array of the data blocks each aux block represents.
|
448
|
+
long aux_block_nr = RANDOM(size);
|
449
|
+
VALUE data_blocks_for_this_aux_block;
|
450
|
+
if ((data_blocks_for_this_aux_block = rb_funcall(aux_ary, get_func, 1, INT2NUM(aux_block_nr))) == Qnil)
|
451
|
+
rb_funcall(aux_ary, set_func, 2, INT2NUM(aux_block_nr), (data_blocks_for_this_aux_block = rb_ary_new()));
|
452
|
+
rb_funcall(data_blocks_for_this_aux_block, push_func, 1, INT2NUM(tmp));
|
453
|
+
}
|
454
|
+
}
|
455
|
+
|
456
|
+
// Then go through all our blocks again.
|
457
|
+
for (tmp = 0; tmp < size; tmp++) {
|
458
|
+
// Fetch the data blocks for this aux block.
|
459
|
+
VALUE data_blocks_for_this_aux_block = rb_funcall(aux_ary, get_func, 1, INT2NUM(tmp));
|
460
|
+
// Count them.
|
461
|
+
long number_of_data_blocks = NUM2INT(rb_funcall(data_blocks_for_this_aux_block, size_func, 0));
|
462
|
+
// Create a String that is 4 times the size.
|
463
|
+
VALUE data_block_string = rb_str_new(0, number_of_data_blocks * 4);
|
464
|
+
// And go through all q blocks per data block, again.
|
465
|
+
for (tmp2 = 0; tmp2 < number_of_data_blocks; tmp2++) {
|
466
|
+
// And make the String contain the data blocks for this aux block, as longs.
|
467
|
+
rb_str_modify(data_block_string);
|
468
|
+
( (long *) RSTRING(data_block_string)->ptr )[tmp2] = (long) NUM2INT(rb_funcall(data_blocks_for_this_aux_block,
|
469
|
+
get_func,
|
470
|
+
1,
|
471
|
+
INT2NUM(tmp2)));
|
472
|
+
}
|
473
|
+
rb_funcall(aux_ary, set_func, 2, INT2NUM(tmp), data_block_string);
|
474
|
+
}
|
475
|
+
rb_ivar_set(self, rb_intern("@_aux_hash"), aux_ary);
|
476
|
+
}
|
477
|
+
EOC
|
478
|
+
#
|
479
|
+
# Try to decode our data and aux blocks until we dont get any more data.
|
480
|
+
#
|
481
|
+
builder.c <<EOC
|
482
|
+
static VALUE
|
483
|
+
_aux_decode(VALUE decode_status, VALUE aux_blocks, VALUE aux_hash)
|
484
|
+
{
|
485
|
+
long tmp, tmp2;
|
486
|
+
char got_new_block = 0;
|
487
|
+
VALUE get_func = rb_intern("[]");
|
488
|
+
|
489
|
+
// Go through all aux blocks.
|
490
|
+
for (tmp = 0; tmp < RSTRING(aux_blocks)->len; tmp++) {
|
491
|
+
// Only check those that are actually known.
|
492
|
+
if (get_is_known(decode_status, RSTRING(self)->len + tmp)) {
|
493
|
+
|
494
|
+
//printf("looking at aux block %d(%d): ", tmp, RSTRING(aux_blocks)->ptr[tmp]);
|
495
|
+
|
496
|
+
// Fetch the String defining what data blocks this aux block is made out of.
|
497
|
+
VALUE data_block_string = rb_funcall(aux_hash, get_func, 1, INT2NUM(tmp));
|
498
|
+
|
499
|
+
// If it is only one data block, then its easy.
|
500
|
+
if (RSTRING(data_block_string)->len == 4) {
|
501
|
+
long block_nr = ( (long *) RSTRING(data_block_string)->ptr )[0];
|
502
|
+
|
503
|
+
//printf(" %d", block_nr);
|
504
|
+
|
505
|
+
// If it is not already known, then make it known.
|
506
|
+
if (!get_is_known(decode_status, block_nr)) {
|
507
|
+
set_value(aux_blocks, block_nr, RSTRING(aux_blocks)->ptr[tmp], self, decode_status);
|
508
|
+
got_new_block = 1;
|
509
|
+
//printf(" => [%d]=%d", block_nr, RSTRING(aux_blocks)->ptr[tmp]);
|
510
|
+
} else {
|
511
|
+
//printf(" [%d]=%d", block_nr, RSTRING(aux_blocks)->ptr[tmp]);
|
512
|
+
}
|
513
|
+
} else {
|
514
|
+
// If it is several data blocks, then we just go through them all to see if only
|
515
|
+
// one of them is unknown.
|
516
|
+
long missing_blocks;
|
517
|
+
long missing_block;
|
518
|
+
// Initialize the xor_sum as this aux_block.
|
519
|
+
char xor_sum = RSTRING(aux_blocks)->ptr[tmp];
|
520
|
+
|
521
|
+
for (missing_blocks = (tmp2 = ((RSTRING(data_block_string)->len / 4) - 1)) + 1; tmp2 > -1; tmp2--) {
|
522
|
+
long block_nr = ( (long *) RSTRING(data_block_string)->ptr )[tmp2];
|
523
|
+
|
524
|
+
//printf(" %d", block_nr);
|
525
|
+
|
526
|
+
// If it is a known data block, decrease missing_blocks and append to the xor_sum.
|
527
|
+
if (get_is_known(decode_status, block_nr)) {
|
528
|
+
missing_blocks--;
|
529
|
+
xor_sum = xor_sum ^ get_value(aux_blocks, block_nr, self);
|
530
|
+
//printf("!(%d)->(%d)", get_value(aux_blocks, block_nr, self), xor_sum);
|
531
|
+
} else {
|
532
|
+
missing_block = block_nr;
|
533
|
+
//printf("?");
|
534
|
+
}
|
535
|
+
}
|
536
|
+
// If only one data block was unknown, then just set it to the built up xor_sum.
|
537
|
+
if (missing_blocks == 1) {
|
538
|
+
set_value(aux_blocks, missing_block, xor_sum, self, decode_status);
|
539
|
+
got_new_block = 1;
|
540
|
+
//printf(" => [%d]=%d", missing_block, xor_sum);
|
541
|
+
}
|
542
|
+
}
|
543
|
+
}
|
544
|
+
}
|
545
|
+
//printf("\\n");
|
546
|
+
fflush(NULL);
|
547
|
+
|
548
|
+
return got_new_block ? Qtrue : Qfalse;
|
549
|
+
}
|
550
|
+
EOC
|
551
|
+
end
|
552
|
+
|
553
|
+
def oneline_encode(chunk_size)
|
554
|
+
ensure_aux_blocks
|
555
|
+
return _get_chunk(chunk_size, @_aux_blocks)
|
556
|
+
end
|
557
|
+
|
558
|
+
def oneline_decode(chunk)
|
559
|
+
|
560
|
+
needed_size = chunk[4..7].unpack("L").first
|
561
|
+
if size == 0
|
562
|
+
concat("\000" * needed_size)
|
563
|
+
elsif size != needed_size
|
564
|
+
raise "The argument to oneline_decode must be a chunk produced by oneline_encode from the same source"
|
565
|
+
end
|
566
|
+
|
567
|
+
@_decode_status ||= "\000" * (((needed_size + get_aux_blocks) / 8) + 1)
|
568
|
+
@_known_chunks ||= []
|
569
|
+
@_aux_blocks ||= "\000" * get_aux_blocks
|
570
|
+
|
571
|
+
found_new_block = _decode_chunk(chunk, @_decode_status, @_aux_blocks)
|
572
|
+
do_decode if found_new_block
|
573
|
+
|
574
|
+
@_known_chunks.unshift(chunk)
|
575
|
+
|
576
|
+
nil while do_decode
|
577
|
+
|
578
|
+
return oneline_done
|
579
|
+
end
|
580
|
+
|
581
|
+
def oneline_done
|
582
|
+
return nil unless @_decode_status
|
583
|
+
return _oneline_done(@_decode_status)
|
584
|
+
end
|
585
|
+
|
586
|
+
def decode_status
|
587
|
+
return nil unless @_decode_status
|
588
|
+
@_decode_status.unpack("b*").first
|
589
|
+
end
|
590
|
+
|
591
|
+
def decode_level
|
592
|
+
ones = 0
|
593
|
+
decode_status[0..(size + @_aux_blocks.size)].split(//).each do |s|
|
594
|
+
ones += 1 if s == "1"
|
595
|
+
end
|
596
|
+
ones.to_f / (size + @_aux_blocks.size).to_f
|
597
|
+
end
|
598
|
+
|
599
|
+
private
|
600
|
+
|
601
|
+
def get_aux_blocks
|
602
|
+
(self.size * (get_q * get_e * 0.55)).ceil
|
603
|
+
end
|
604
|
+
|
605
|
+
def get_q
|
606
|
+
0.upto(THRESHOLDS.size - 1) do |n|
|
607
|
+
if size < THRESHOLDS[n]
|
608
|
+
return Q[n]
|
609
|
+
end
|
610
|
+
end
|
611
|
+
return Q.last
|
612
|
+
end
|
613
|
+
|
614
|
+
def do_decode
|
615
|
+
found_new_block = false
|
616
|
+
@_known_chunks.each do |known_chunk|
|
617
|
+
found_new_block = _decode_chunk(known_chunk, @_decode_status, @_aux_blocks) || found_new_block
|
618
|
+
end
|
619
|
+
ensure_aux_hash
|
620
|
+
found_new_block = _aux_decode(@_decode_status, @_aux_blocks, @_aux_hash) || found_new_block
|
621
|
+
return found_new_block
|
622
|
+
end
|
623
|
+
|
624
|
+
def get_e
|
625
|
+
0.upto(THRESHOLDS.size - 1) do |n|
|
626
|
+
if size < THRESHOLDS[n]
|
627
|
+
return E[n]
|
628
|
+
end
|
629
|
+
end
|
630
|
+
return E.last
|
631
|
+
end
|
632
|
+
|
633
|
+
def ensure_aux_blocks
|
634
|
+
_fill_aux_blocks(get_aux_blocks, get_q) unless @_aux_blocks
|
635
|
+
end
|
636
|
+
|
637
|
+
def ensure_aux_hash
|
638
|
+
_fill_aux_hash(get_q, get_aux_blocks) unless @_aux_hash
|
639
|
+
end
|
640
|
+
|
641
|
+
end
|
data/lib/archipelago/pirate.rb
CHANGED
@@ -18,6 +18,7 @@
|
|
18
18
|
require 'archipelago/disco'
|
19
19
|
require 'archipelago/tranny'
|
20
20
|
require 'archipelago/treasure'
|
21
|
+
require 'archipelago/client'
|
21
22
|
require 'pp'
|
22
23
|
require 'drb'
|
23
24
|
require 'digest/sha1'
|
@@ -60,8 +61,6 @@ module Archipelago
|
|
60
61
|
end
|
61
62
|
end
|
62
63
|
|
63
|
-
INITIAL_SERVICE_UPDATE_INTERVAL = 1
|
64
|
-
MAXIMUM_SERVICE_UPDATE_INTERVAL = 60
|
65
64
|
CHEST_DESCRIPTION = {
|
66
65
|
:class => 'Archipelago::Treasure::Chest'
|
67
66
|
}
|
@@ -73,12 +72,8 @@ module Archipelago
|
|
73
72
|
# The class that actually keeps track of the Archipelago::Treasure:Chests and the
|
74
73
|
# Archipelago::Treasure:Dubloons in them.
|
75
74
|
#
|
76
|
-
class Captain
|
77
|
-
attr_reader :chests, :
|
78
|
-
#
|
79
|
-
# Initialize an instance using an Archipelago::Disco::Jockey with <i>:jockey_options</i>
|
80
|
-
# if given, that looks for new services <i>:initial_service_update_interval</i> or INITIAL_SERVICE_UPDATE_INTERVAL,
|
81
|
-
# when it starts and never slower than every <i>:maximum_service_update_interval</i> or MAXIMUM_SERVICE_UPDATE_INTERVAL.
|
75
|
+
class Captain < Archipelago::Client::Base
|
76
|
+
attr_reader :chests, :trannies
|
82
77
|
#
|
83
78
|
# Will look for Archipelago::Treasure::Chests matching <i>:chest_description</i> or CHEST_DESCRIPTION and
|
84
79
|
# Archipelago::Tranny::Managers matching <i>:tranny_description</i> or TRANNY_DESCRIPTION.
|
@@ -87,12 +82,11 @@ module Archipelago
|
|
87
82
|
# of required classes and modules at the chest.
|
88
83
|
#
|
89
84
|
def initialize(options = {})
|
90
|
-
|
85
|
+
super(options)
|
91
86
|
|
92
87
|
@transaction = nil
|
93
88
|
|
94
|
-
start_service_updater
|
95
|
-
options[:maximum_service_update_interval] || MAXIMUM_SERVICE_UPDATE_INTERVAL)
|
89
|
+
start_service_updater
|
96
90
|
|
97
91
|
end
|
98
92
|
|
@@ -100,14 +94,7 @@ module Archipelago
|
|
100
94
|
# Sets up this instance with the given +options+.
|
101
95
|
#
|
102
96
|
def setup(options = {})
|
103
|
-
|
104
|
-
if defined?(Archipelago::Disco::MC)
|
105
|
-
@treasure_map.stop! if @treasure_map && @treasure_map != Archipelago::Disco::MC
|
106
|
-
@treasure_map = options.include?(:jockey_options) ? Archipelago::Disco::Jockey.new(options[:jockey_options]) : Archipelago::Disco::MC
|
107
|
-
else
|
108
|
-
@treasure_map.stop! if @treasure_map
|
109
|
-
@treasure_map = Archipelago::Disco::Jockey.new(options[:jockey_options] || {})
|
110
|
-
end
|
97
|
+
super(options)
|
111
98
|
|
112
99
|
@chest_description = CHEST_DESCRIPTION.merge(options[:chest_description] || {})
|
113
100
|
@tranny_description = TRANNY_DESCRIPTION.merge(options[:tranny_description] || {})
|
@@ -193,8 +180,11 @@ module Archipelago
|
|
193
180
|
|
194
181
|
@transaction = @trannies.values.first[:service].begin
|
195
182
|
begin
|
196
|
-
|
197
|
-
|
183
|
+
begin
|
184
|
+
return yield(@transaction)
|
185
|
+
ensure
|
186
|
+
raise CommitFailedException.new(self, @transaction) unless @transaction.commit! == :commited
|
187
|
+
end
|
198
188
|
rescue Exception => e
|
199
189
|
@transaction.abort! unless @transaction.state == :aborted
|
200
190
|
raise e
|
@@ -235,17 +225,6 @@ module Archipelago
|
|
235
225
|
'yar!'
|
236
226
|
end
|
237
227
|
|
238
|
-
#
|
239
|
-
# Stops the service update thread for this Pirate and also unpublishes and/or
|
240
|
-
# stops the Jockey.
|
241
|
-
#
|
242
|
-
def stop!
|
243
|
-
@service_update_thread.kill
|
244
|
-
unless defined?(Archipelago::Disco::MC) && @treasure_map && @treasure_map == Archipelago::Disco::MC
|
245
|
-
@treasure_map.stop!
|
246
|
-
end
|
247
|
-
end
|
248
|
-
|
249
228
|
#
|
250
229
|
# Will do +callable+.call(key, value)
|
251
230
|
# for each key-and-value pair in this database network.
|
@@ -263,9 +242,9 @@ module Archipelago
|
|
263
242
|
# Does an immediate update of our service lists.
|
264
243
|
#
|
265
244
|
def update_services!
|
266
|
-
@chests = @
|
245
|
+
@chests = @jockey.lookup(Archipelago::Disco::Query.new(@chest_description), 0)
|
267
246
|
evaluate_in_chests
|
268
|
-
@trannies = @
|
247
|
+
@trannies = @jockey.lookup(Archipelago::Disco::Query.new(@tranny_description), 0)
|
269
248
|
end
|
270
249
|
|
271
250
|
private
|
@@ -329,30 +308,16 @@ module Archipelago
|
|
329
308
|
end
|
330
309
|
end
|
331
310
|
|
332
|
-
#
|
333
|
-
# Start a thread looking up existing chests between every
|
334
|
-
# +initial+ and +maximum+ seconds.
|
335
|
-
#
|
336
|
-
def start_service_updater(initial, maximum)
|
337
|
-
update_services!
|
338
|
-
@service_update_thread = Thread.start do
|
339
|
-
standoff = initial
|
340
|
-
loop do
|
341
|
-
begin
|
342
|
-
sleep(standoff)
|
343
|
-
standoff *= 2
|
344
|
-
standoff = maximum if standoff > maximum
|
345
|
-
update_services!
|
346
|
-
rescue Exception => e
|
347
|
-
puts e
|
348
|
-
pp e.backtrace
|
349
|
-
end
|
350
|
-
end
|
351
|
-
end
|
352
|
-
end
|
353
|
-
|
354
311
|
end
|
355
312
|
|
313
|
+
#
|
314
|
+
# The default Archipelago::Pirate::Captain that is always available for lookups is Archipelago::Pirate::BLACKBEARD.
|
315
|
+
#
|
316
|
+
# If you really need to you can customize it by defining BLACKBEARD_OPTIONS before loading disco.rb, and if you REALLY
|
317
|
+
# need to you can disable it completely by setting BLACKBEARD_DISABLED to true.
|
318
|
+
#
|
319
|
+
BLACKBEARD = Captain.new(defined?(BLACKBEARD_OPTIONS) ? BLACKBEARD_OPTIONS : {}) unless defined?(BLACKBEARD_DISABLED) && BLACKBEARD_DISABLED
|
320
|
+
|
356
321
|
end
|
357
322
|
|
358
323
|
end
|
data/lib/archipelago.rb
CHANGED
@@ -17,8 +17,14 @@
|
|
17
17
|
|
18
18
|
$: << File.dirname(File.expand_path(__FILE__))
|
19
19
|
|
20
|
+
require 'archipelago/bitstring'
|
21
|
+
require 'archipelago/oneliner'
|
22
|
+
require 'archipelago/raider'
|
20
23
|
require 'archipelago/disco'
|
21
24
|
require 'archipelago/current'
|
22
25
|
require 'archipelago/tranny'
|
23
26
|
require 'archipelago/treasure'
|
27
|
+
require 'archipelago/client'
|
24
28
|
require 'archipelago/pirate'
|
29
|
+
require 'archipelago/cove'
|
30
|
+
require 'archipelago/exxon'
|
data/script/services.rb
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
if ARGV.size < 1
|
4
|
-
puts "Usage: #{$0} DB-PATH [EXTRA-LIBS]"
|
4
|
+
puts "Usage: #{$0} DB-PATH [DRB-URI [EXTRA-LIBS]]"
|
5
5
|
exit 1
|
6
6
|
end
|
7
7
|
|
@@ -9,16 +9,22 @@ $: << File.join(File.dirname(__FILE__), "..", "lib")
|
|
9
9
|
|
10
10
|
require 'archipelago/treasure'
|
11
11
|
require 'archipelago/tranny'
|
12
|
+
require 'archipelago/cove'
|
12
13
|
|
13
|
-
ARGV
|
14
|
-
|
14
|
+
if ARGV.size > 1
|
15
|
+
DRb.start_service(ARGV[1])
|
16
|
+
ARGV[2..-1].each do |lib|
|
17
|
+
load lib
|
18
|
+
end
|
19
|
+
else
|
20
|
+
DRb.start_service
|
15
21
|
end
|
16
22
|
|
17
|
-
DRb.start_service
|
18
|
-
|
19
23
|
t = Archipelago::Tranny::Manager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tranny"))))
|
20
24
|
t.publish!
|
21
25
|
c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "chest"))))
|
22
26
|
c.publish!
|
27
|
+
tank = Archipelago::Cove::Tanker.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.join(ARGV[0], "tanker"))))
|
28
|
+
tank.publish!
|
23
29
|
|
24
30
|
DRb.thread.join
|
data/tests/test_helper.rb
CHANGED
metadata
CHANGED
@@ -3,8 +3,8 @@ rubygems_version: 0.8.11
|
|
3
3
|
specification_version: 1
|
4
4
|
name: archipelago
|
5
5
|
version: !ruby/object:Gem::Version
|
6
|
-
version: 0.2.
|
7
|
-
date: 2006-12-
|
6
|
+
version: 0.2.5
|
7
|
+
date: 2006-12-20 00:00:00 +01:00
|
8
8
|
summary: A set of tools for distributed computing in ruby.
|
9
9
|
require_paths:
|
10
10
|
- lib
|
@@ -29,9 +29,13 @@ authors:
|
|
29
29
|
- Martin Kihlgren
|
30
30
|
files:
|
31
31
|
- lib/archipelago.rb
|
32
|
+
- lib/archipelago/client.rb
|
33
|
+
- lib/archipelago/cove.rb
|
32
34
|
- lib/archipelago/current.rb
|
33
35
|
- lib/archipelago/disco.rb
|
36
|
+
- lib/archipelago/exxon.rb
|
34
37
|
- lib/archipelago/hashish.rb
|
38
|
+
- lib/archipelago/oneline.rb
|
35
39
|
- lib/archipelago/pirate.rb
|
36
40
|
- lib/archipelago/tranny.rb
|
37
41
|
- lib/archipelago/treasure.rb
|
@@ -46,12 +50,10 @@ files:
|
|
46
50
|
- tests/tranny_test.rb
|
47
51
|
- tests/treasure_benchmark.rb
|
48
52
|
- tests/treasure_test.rb
|
49
|
-
- script/chest.rb
|
50
53
|
- script/console
|
51
54
|
- script/overloads.rb
|
52
55
|
- script/pirate.rb
|
53
56
|
- script/services.rb
|
54
|
-
- script/tranny.rb
|
55
57
|
- GPL-2
|
56
58
|
- TODO
|
57
59
|
- README
|
data/script/chest.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
if ARGV.size < 1
|
4
|
-
puts "Usage: #{$0} DB-PATH [EXTRA-LIBS]"
|
5
|
-
exit 1
|
6
|
-
end
|
7
|
-
|
8
|
-
$: << File.join(File.dirname(__FILE__), "..", "lib")
|
9
|
-
|
10
|
-
require 'archipelago/treasure'
|
11
|
-
|
12
|
-
ARGV[1..-1].each do |lib|
|
13
|
-
load lib
|
14
|
-
end
|
15
|
-
|
16
|
-
DRb.start_service
|
17
|
-
|
18
|
-
c = Archipelago::Treasure::Chest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(ARGV[0])))
|
19
|
-
c.publish!
|
20
|
-
|
21
|
-
DRb.thread.join
|
data/script/tranny.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
if ARGV.size < 1
|
4
|
-
puts "Usage: #{$0} DB-PATH [EXTRA-LIBS]"
|
5
|
-
exit 1
|
6
|
-
end
|
7
|
-
|
8
|
-
$: << File.join(File.dirname(__FILE__), "..", "lib")
|
9
|
-
|
10
|
-
require 'archipelago/treasure'
|
11
|
-
|
12
|
-
ARGV[1..-1].each do |lib|
|
13
|
-
load lib
|
14
|
-
end
|
15
|
-
|
16
|
-
DRb.start_service
|
17
|
-
|
18
|
-
t = Archipelago::Tranny::Manager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(ARGV[0])))
|
19
|
-
t.publish!
|
20
|
-
|
21
|
-
DRb.thread.join
|