archipelago 0.2.5 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/tests/test_helper.rb CHANGED
@@ -2,17 +2,25 @@
2
2
  home = File.expand_path(File.dirname(__FILE__))
3
3
  $: << File.join(home, "..", "lib")
4
4
 
5
- MC_DISABLED = true unless defined?(MC_ENABLED) && MC_ENABLED
6
- BLACKBEARD_DISABLED = true unless defined?(BLACKBEARD_ENABLED) && BLACKBEARD_ENABLED
5
+ MC_DISABLED = true
6
+ CLEANER_DISABLED = true
7
+ BLACKBEARD_DISABLED = true
7
8
 
8
- require 'pp'
9
9
  require 'drb'
10
+ DRb.start_service
11
+
12
+ require 'pp'
10
13
  require 'test/unit'
11
14
  require 'archipelago'
15
+ require 'archipelago/treasure'
16
+ require 'archipelago/pirate'
17
+ require 'archipelago/dump'
12
18
  require 'benchmark'
13
19
  require 'socket'
14
20
  require 'ipaddr'
15
21
  require 'thread'
22
+ require 'rubygems'
23
+ require 'timeout'
16
24
 
17
25
  class TestTransaction
18
26
  def join(o)
@@ -36,6 +44,10 @@ end
36
44
 
37
45
  class TestChest < Archipelago::Treasure::Chest
38
46
  attr_reader :persistence_provider
47
+ alias :old_assert_mine :assert_mine
48
+ def assert_mine(key)
49
+ old_assert_mine(key) if defined?($DO_ASSERT_MINE) && $DO_ASSERT_MINE
50
+ end
39
51
  end
40
52
 
41
53
  def bm(label = "", options = {})
data/tests/tranny_test.rb CHANGED
@@ -4,13 +4,11 @@ require File.join(File.dirname(__FILE__), 'test_helper')
4
4
  class TrannyTest < Test::Unit::TestCase
5
5
 
6
6
  def setup
7
- DRb.start_service
8
7
  @tm = TestManager.new(:db_path => Pathname.new(__FILE__).parent.join("tranny.db"))
9
8
  end
10
9
 
11
10
  def teardown
12
11
  @tm.persistence_provider.unlink!
13
- DRb.stop_service
14
12
  end
15
13
 
16
14
  class Participant
@@ -4,13 +4,16 @@ require File.join(File.dirname(__FILE__), 'test_helper')
4
4
  class TreasureBenchmark < Test::Unit::TestCase
5
5
 
6
6
  def setup
7
- DRb.start_service
8
- @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db")))
7
+ @p = Archipelago::Pirate::Captain.new(:chest_description => {:class => "TestChest"},
8
+ :tranny_description => {:class => "TestManager"},
9
+ :initial_lookup_timeout => 0)
10
+ @c = TestChest.new(:captain => @p,
11
+ :persistence_directory => Pathname.new(__FILE__).parent.join("chest.db"))
9
12
  end
10
13
 
11
14
  def teardown
12
15
  @c.persistence_provider.unlink!
13
- DRb.stop_service
16
+ @p.stop!
14
17
  end
15
18
 
16
19
  def test_outside_transaction
@@ -23,10 +23,15 @@ end
23
23
  class TreasureTest < Test::Unit::TestCase
24
24
 
25
25
  def setup
26
- DRb.start_service
27
- @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db")))
28
- @c2 = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest2.db")))
29
- @tm = TestManager.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("tranny1.db")))
26
+ @p = Archipelago::Pirate::Captain.new(:chest_description => {:class => "TestChest"},
27
+ :tranny_description => {:class => "TestManager"},
28
+ :initial_lookup_timeout => 0)
29
+ @c = TestChest.new(:captain => @p,
30
+ :persistence_directory => Pathname.new(__FILE__).parent.join("chest.db"))
31
+ @c2 = TestChest.new(:captain => @p,
32
+ :persistence_directory => Pathname.new(__FILE__).parent.join("chest2.db"))
33
+ @tm = TestManager.new(:captain => @p,
34
+ :persistence_directory => Pathname.new(__FILE__).parent.join("tranny1.db"))
30
35
  $BURKMAT = 0
31
36
  $BURKMAT2 = 0
32
37
  $BURKMAT3 = 0
@@ -37,10 +42,13 @@ class TreasureTest < Test::Unit::TestCase
37
42
  end
38
43
 
39
44
  def teardown
45
+ @c.close!
40
46
  @c.persistence_provider.unlink!
47
+ @c2.close!
41
48
  @c2.persistence_provider.unlink!
49
+ @tm.close!
42
50
  @tm.persistence_provider.unlink!
43
- DRb.stop_service
51
+ @p.stop!
44
52
  end
45
53
 
46
54
  def test_with_transaction
@@ -87,12 +95,12 @@ class TreasureTest < Test::Unit::TestCase
87
95
  def test_around_load
88
96
  @c["brunis"] = A.new("hirr")
89
97
  @c.persistence_provider.close!
90
- @c = TestChest.new(:persistence_provider => Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(__FILE__).parent.join("chest.db")))
98
+ @c = TestChest.new(:persistence_directory => Pathname.new(__FILE__).parent.join("chest.db"), :captain => @p)
91
99
  a = @c["brunis"]
92
100
  assert_equal(1, $BURKMAT3)
93
101
  assert_equal(1, $BURKMAT4)
94
102
  end
95
-
103
+
96
104
  def test_around_save
97
105
  s = A.new("hehu")
98
106
  @c["oj"] = s
@@ -341,6 +349,34 @@ class TreasureTest < Test::Unit::TestCase
341
349
  end
342
350
  assert(@c.active_transactions.empty?)
343
351
  end
352
+
353
+ def test_transaction_locking_and_recovery
354
+ t = @tm.begin
355
+ @c["oj", t] = "blar"
356
+ @c2["bga", t] = "kggkgk"
357
+ @c.prepare!(t)
358
+ @c2.prepare!(t)
359
+ assert_raise(Timeout::Error) do
360
+ timeout(1) do
361
+ Thread.new do
362
+ @c2["bga"] = "ojoj"
363
+ end.join
364
+ end
365
+ end
366
+ @c2.close!
367
+ @c2 = TestChest.new(:captain => @p,
368
+ :persistence_directory => Pathname.new(__FILE__).parent.join("chest2.db"))
369
+ assert(@c2.instance_eval do
370
+ @snapshot_by_transaction[t].include?("bga")
371
+ end)
372
+ assert_raise(Timeout::Error) do
373
+ timeout(1) do
374
+ Thread.new do
375
+ @c2["bga"] = "ojoj"
376
+ end.join
377
+ end
378
+ end
379
+ end
344
380
 
345
381
  private
346
382
 
metadata CHANGED
@@ -1,10 +1,10 @@
1
1
  --- !ruby/object:Gem::Specification
2
- rubygems_version: 0.8.11
2
+ rubygems_version: 0.9.2
3
3
  specification_version: 1
4
4
  name: archipelago
5
5
  version: !ruby/object:Gem::Version
6
- version: 0.2.5
7
- date: 2006-12-20 00:00:00 +01:00
6
+ version: 0.2.6
7
+ date: 2007-05-14 00:00:00 +02:00
8
8
  summary: A set of tools for distributed computing in ruby.
9
9
  require_paths:
10
10
  - lib
@@ -25,32 +25,36 @@ required_ruby_version: !ruby/object:Gem::Version::Requirement
25
25
  platform: ruby
26
26
  signing_key:
27
27
  cert_chain:
28
+ post_install_message:
28
29
  authors:
29
30
  - Martin Kihlgren
30
31
  files:
31
32
  - lib/archipelago.rb
32
33
  - lib/archipelago/client.rb
33
- - lib/archipelago/cove.rb
34
34
  - lib/archipelago/current.rb
35
35
  - lib/archipelago/disco.rb
36
- - lib/archipelago/exxon.rb
36
+ - lib/archipelago/dump.rb
37
37
  - lib/archipelago/hashish.rb
38
- - lib/archipelago/oneline.rb
39
38
  - lib/archipelago/pirate.rb
39
+ - lib/archipelago/sanitation.rb
40
40
  - lib/archipelago/tranny.rb
41
41
  - lib/archipelago/treasure.rb
42
42
  - tests/current_benchmark.rb
43
43
  - tests/current_test.rb
44
44
  - tests/disco_benchmark.rb
45
45
  - tests/disco_test.rb
46
+ - tests/dump_test.rb
46
47
  - tests/evaltest
47
48
  - tests/evaltestmore
48
49
  - tests/pirate_test.rb
50
+ - tests/sanitation_benchmark.rb
51
+ - tests/sanitation_test.rb
49
52
  - tests/test_helper.rb
50
53
  - tests/tranny_test.rb
51
54
  - tests/treasure_benchmark.rb
52
55
  - tests/treasure_test.rb
53
- - script/console
56
+ - script/console.rb
57
+ - script/officer.rb
54
58
  - script/overloads.rb
55
59
  - script/pirate.rb
56
60
  - script/services.rb
@@ -60,7 +64,9 @@ files:
60
64
  test_files:
61
65
  - tests/current_test.rb
62
66
  - tests/disco_test.rb
67
+ - tests/dump_test.rb
63
68
  - tests/pirate_test.rb
69
+ - tests/sanitation_test.rb
64
70
  - tests/tranny_test.rb
65
71
  - tests/treasure_test.rb
66
72
  - tests/test_helper.rb
@@ -1,68 +0,0 @@
1
- # Archipelago - a distributed computing toolkit for ruby
2
- # Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
3
- #
4
- # This program is free software; you can redistribute it and/or
5
- # modify it under the terms of the GNU General Public License
6
- # as published by the Free Software Foundation; either version 2
7
- # of the License, or (at your option) any later version.
8
- #
9
- # This program is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
- # GNU General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU General Public License
15
- # along with this program; if not, write to the Free Software
16
- # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17
-
18
- require 'archipelago/disco'
19
- require 'archipelago/hashish'
20
- require 'archipelago/exxon'
21
- require 'archipelago/raider'
22
-
23
- module Archipelago
24
-
25
- module Cove
26
-
27
- class Tanker
28
-
29
- #
30
- # The Tanker can be published.
31
- #
32
- include Archipelago::Disco::Publishable
33
-
34
- def initialize(options = {})
35
- #
36
- # The provider of happy magic persistent hashes of different kinds.
37
- #
38
- @persistence_provider = options[:persistence_provider] || Archipelago::Hashish::BerkeleyHashishProvider.new(Pathname.new(File.expand_path(__FILE__)).parent.join("cove_tanker.db"))
39
-
40
- @db = @persistence_provider.get_hashish("db")
41
-
42
- @valdez = options[:valdez] || Archipelago::Exxon::HAZELWOOD
43
-
44
- #
45
- # Use the given options to initialize the publishable
46
- # instance variables.
47
- #
48
- initialize_publishable(options)
49
-
50
- end
51
-
52
- def store_part(key, split_part)
53
- @db[key] = Marshal.dump(split_part)
54
- end
55
-
56
- def get_part(key)
57
- if @db.include?(key)
58
- return Marshal.load(@db[key])
59
- else
60
- return nil
61
- end
62
- end
63
-
64
- end
65
-
66
- end
67
-
68
- end
@@ -1,138 +0,0 @@
1
- # Archipelago - a distributed computing toolkit for ruby
2
- # Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
3
- #
4
- # This program is free software; you can redistribute it and/or
5
- # modify it under the terms of the GNU General Public License
6
- # as published by the Free Software Foundation; either version 2
7
- # of the License, or (at your option) any later version.
8
- #
9
- # This program is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
- # GNU General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU General Public License
15
- # along with this program; if not, write to the Free Software
16
- # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17
-
18
- require 'archipelago/client'
19
-
20
- module Archipelago
21
-
22
- module Exxon
23
-
24
- TANKER_DESCRIPTION = {
25
- :class => 'Archipelago::Cove::Tanker'
26
- }
27
-
28
- PARTS = 3
29
-
30
- #
31
- # Raised when you try to do stuff without any remote database
32
- # available.
33
- #
34
- class NoRemoteDatabaseAvailableException < RuntimeError
35
- def initialize(valdez)
36
- super("#{valdez} can not find any remote database for you")
37
- end
38
- end
39
-
40
- class Valdez < Archipelago::Client::Base
41
- attr_reader :tankers
42
- def initialize(options = {})
43
- super(options)
44
-
45
- start_service_updater
46
- end
47
-
48
- def setup(options = {})
49
- super(options)
50
-
51
- @parts = options[:parts] || PARTS
52
- @tanker_description = TANKER_DESCRIPTION.merge(options[:tanker_description] || {})
53
- end
54
-
55
- def []=(key, value)
56
- split_parts = Archipelago::Raider.split(value, @parts)
57
- followers = responsible_tankers(key, split_parts.size)
58
- 0.upto(followers.size - 1) do |n|
59
- followers[n].store_part(key, split_parts[n])
60
- end
61
- end
62
-
63
- def [](key)
64
- return_value = nil
65
-
66
- split_parts = []
67
- followers = responsible_tankers(key, (1 << @parts) - 1)
68
- while split_parts.size < @parts
69
- split_parts << followers.shift.get_part(key)
70
- end
71
- begin
72
- return_value = Archipelago::Raider.join(*split_parts.compact)
73
- rescue Archipelago::Raider::NotEnoughBulkError => e
74
- if followers.empty?
75
- raise e
76
- else
77
- split_parts << followers.shift.get_part(key)
78
- retry
79
- end
80
- end
81
-
82
- return return_value
83
- end
84
-
85
- def update_services!
86
- @tankers = @jockey.lookup(Archipelago::Disco::Query.new(@tanker_description), 0)
87
- end
88
-
89
- def responsible_tankers(key, n)
90
- raise NoRemoteDatabaseAvailableException.new(self) if @tankers.empty?
91
-
92
- responsible_ids = []
93
-
94
- key_id = Digest::SHA1.hexdigest(Marshal.dump(key))
95
- sorted_tanker_ids = @tankers.keys.sort
96
- 0.upto(sorted_tanker_ids.size - 1) do |index|
97
- id = sorted_tanker_ids[index]
98
- if id > key_id
99
- responsible_ids = get_some(sorted_tanker_ids, index, n)
100
- end
101
- end
102
- responsible_ids = get_some(sorted_tanker_ids, 0, n)
103
-
104
- return responsible_ids.collect do |id|
105
- @tankers[id][:service]
106
- end
107
- end
108
-
109
- def responsible_tanker(key)
110
- return responsible_tankers(key, 1).first
111
- end
112
-
113
- private
114
-
115
- def get_some(array, index, n)
116
- rval = []
117
- while rval.size < n
118
- index = 0 if index > array.size - 1
119
- rval << array[index]
120
- index += 1
121
- end
122
- return rval
123
- end
124
-
125
- end
126
-
127
- #
128
- # The default Archipelago::Exxon::Valdez that is always available for lookups is Archipelago::Exxon::HAZELWOOD.
129
- #
130
- # If you really need to you can customize it by defining HAZELWOOD_OPTIONS before loading disco.rb, and if you REALLY
131
- # need to you can disable it completely by setting HAZELWOOD_DISABLED to true.
132
- #
133
- HAZELWOOD = Valdez.new(defined?(HAZELWOOD_OPTIONS) ? HAZELWOOD_OPTIONS : {}) unless defined?(HAZELWOOD_DISABLED) && HAZELWOOD_DISABLED
134
-
135
-
136
- end
137
-
138
- end
@@ -1,641 +0,0 @@
1
- # Archipelago - a distributed computing toolkit for ruby
2
- # Copyright (C) 2006 Martin Kihlgren <zond at troja dot ath dot cx>
3
- #
4
- # This program is free software; you can redistribute it and/or
5
- # modify it under the terms of the GNU General Public License
6
- # as published by the Free Software Foundation; either version 2
7
- # of the License, or (at your option) any later version.
8
- #
9
- # This program is distributed in the hope that it will be useful,
10
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
11
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12
- # GNU General Public License for more details.
13
- #
14
- # You should have received a copy of the GNU General Public License
15
- # along with this program; if not, write to the Free Software
16
- # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
17
-
18
- require 'rubygems'
19
- require 'inline'
20
- require 'pp'
21
-
22
- #
23
- # Adds a few methods to String, some using the inline gem.
24
- #
25
- class String
26
-
27
- E = [0.1, 0.01]
28
- Q = [6, 3]
29
- THRESHOLDS = [1024]
30
-
31
- class << self
32
- private
33
- #
34
- # Get an F value for given epsilon value
35
- #
36
- def get_f(e)
37
- Math.log(e ** 2 / 4) / Math.log(1 - (e / 2))
38
- end
39
- #
40
- # Get a p distribution with a given name and epsilon value.
41
- #
42
- # Will return a C variable definition for this distribution.
43
- #
44
- def _get_p_distribution(name, e)
45
- f = get_f(e)
46
- rval = "double #{name}[] = {0.0,"
47
- p1 = 1 - ( (1 + (1 / f) ) / (1 + e) )
48
- 1.upto(f) do |n|
49
- if n == 1
50
- rval << "#{p1}"
51
- else
52
- rval << "#{( ( 1 - p1 ) * f ) / ( ( f - 1 ) * n * ( n - 1 ) )}"
53
- end
54
- rval << "," unless n + 1 > f
55
- end
56
- rval << "};\n"
57
- return rval
58
- end
59
- #
60
- # Get all the p distributions we want.
61
- #
62
- # Will return a C variable definition for this distribution.
63
- #
64
- def get_p_distribution
65
- rval = ""
66
- 0.upto(E.size - 1) do |n|
67
- rval << _get_p_distribution("P#{n}", E[n])
68
- end
69
- return rval
70
- end
71
- end
72
-
73
- inline do |builder|
74
- #
75
- # We need math.h
76
- #
77
- builder.include("<math.h>")
78
- #
79
- # We need sys/time.h
80
- #
81
- builder.include("<sys/time.h>")
82
- #
83
- # Define the p distribution array so that it is easily accessible.
84
- #
85
- builder.prefix(get_p_distribution)
86
- #
87
- # Define a sane random number generator.
88
- #
89
- builder.prefix("#define RAND (((double)random())/((double)RAND_MAX+1))")
90
- builder.prefix("#define RANDOM(n) (random() % (n))")
91
- #
92
- # Get a random degree.
93
- #
94
- # Will define as many get_degree{n} functions as we need.
95
- #
96
- degree_methods = ""
97
- 0.upto(E.size - 1) do |n|
98
- degree_methods += <<EOC
99
- static long
100
- get_degree#{n}(size)
101
- {
102
- long rval = 1;
103
- double d = RAND - P#{n}[rval];
104
- while (d > 0.0) {
105
- rval ++;
106
- d = d - P#{n}[rval];
107
- }
108
- if (rval > size)
109
- return size;
110
- else
111
- return rval;
112
- }
113
- EOC
114
- end
115
- builder.prefix(degree_methods)
116
- #
117
- # Get the function to produce random degrees for us.
118
- #
119
- # Will go through our THRESHOLDS and return the proper degree-method.
120
- #
121
- get_degree_func = <<EOC
122
- long (*get_degree_func(long size))(long)
123
- {
124
- EOC
125
- 0.upto(THRESHOLDS.size - 1) do |n|
126
- get_degree_func += <<EOC
127
- if (size < #{THRESHOLDS[n]})
128
- return &get_degree#{n};
129
- EOC
130
- end
131
- get_degree_func += <<EOC
132
- return &get_degree#{THRESHOLDS.size};
133
- EOC
134
- get_degree_func += <<EOC
135
- }
136
- EOC
137
- builder.prefix(get_degree_func)
138
- #
139
- # Check if a certain block is known.
140
- #
141
- builder.prefix <<EOC
142
- static char
143
- get_is_known(VALUE decode_status, long block_number)
144
- {
145
- long byte_in_status = block_number / 8;
146
- char bit_in_byte = 1 << (block_number % 8);
147
- return ((RSTRING(decode_status)->ptr[byte_in_status]) & bit_in_byte) == bit_in_byte;
148
- }
149
- EOC
150
- #
151
- # Set that a certain block is known.
152
- #
153
- builder.prefix <<EOC
154
- static void
155
- set_is_known(VALUE decode_status, long block_number)
156
- {
157
- long byte_in_status = block_number / 8;
158
- char bit_in_byte = 1 << (block_number % 8);
159
- rb_str_modify(decode_status);
160
- RSTRING(decode_status)->ptr[byte_in_status] = RSTRING(decode_status)->ptr[byte_in_status] | bit_in_byte;
161
- }
162
- EOC
163
- #
164
- # Returns whether this String is decoded enough.
165
- #
166
- # Will first go through our decode_status one byte a time
167
- # and then check the relevant bits of the last byte.
168
- #
169
- builder.c <<EOC
170
- static VALUE
171
- _oneline_done(VALUE decode_status)
172
- {
173
- long tmp;
174
- if (decode_status == Qnil)
175
- return Qfalse;
176
- for (tmp = 0; tmp < (RSTRING(decode_status)->len - 1); tmp++) {
177
- if (( (unsigned char) RSTRING(decode_status)->ptr[tmp] ) != 255) {
178
- return Qfalse;
179
- }
180
- }
181
- for (tmp = 0; tmp < (RSTRING(self)->len % 8); tmp++) {
182
- if (! ( (unsigned char) RSTRING(self)->ptr[tmp] ) & (1 << tmp)) {
183
- return Qfalse;
184
- }
185
- }
186
- return Qtrue;
187
- }
188
- EOC
189
- #
190
- # Set a data or aux block with a known value.
191
- #
192
- # Will both update ourselves or our aux_blocks depending on index,
193
- # then set the decode_status so that this byte is officially known.
194
- #
195
- builder.prefix <<EOC
196
- static void
197
- set_value(VALUE aux_blocks, long index, char value, VALUE self, VALUE decode_status)
198
- {
199
- if (index < RSTRING(self)->len) {
200
- rb_str_modify(self);
201
- RSTRING(self)->ptr[index] = value;
202
- set_is_known(decode_status, index);
203
- } else {
204
- rb_str_modify(aux_blocks);
205
- RSTRING(aux_blocks)->ptr[index - RSTRING(self)->len] = value;
206
- set_is_known(decode_status, index);
207
- }
208
- }
209
- EOC
210
- #
211
- # Get a data or aux block.
212
- #
213
- # Gets from ourselves or our aux_blocks depending on the index.
214
- #
215
- builder.prefix <<EOC
216
- static char
217
- get_value(VALUE aux_blocks, long index, VALUE self)
218
- {
219
- if (index < RSTRING(self)->len) {
220
- return RSTRING(self)->ptr[index];
221
- } else {
222
- return RSTRING(aux_blocks)->ptr[index - RSTRING(self)->len];
223
- }
224
- }
225
- EOC
226
- #
227
- # Decode the given chunk until we dont get any new
228
- # data from it.
229
- #
230
- builder.c <<EOC
231
- static VALUE
232
- _decode_chunk(VALUE chunk, VALUE decode_status, VALUE aux_blocks)
233
- {
234
- long seed = ( (long *) RSTRING(chunk)->ptr )[0];
235
- long size = ( (long *) RSTRING(chunk)->ptr )[1];
236
- long tmp, tmp2;
237
- char got_new_block = 0;
238
- long (*degree_func)() = get_degree_func(size);
239
-
240
- // Go through the chunk.
241
- for (tmp = 8; tmp < RSTRING(chunk)->len; tmp++) {
242
- long degree;
243
-
244
- // Seed with our seed and the index in the chunk.
245
- srandom(seed + tmp);
246
-
247
- //printf("looking at check block %d(%d):", tmp, RSTRING(chunk)->ptr[tmp]);
248
-
249
- // Get the degree of this chunk.
250
- degree = degree_func(size);
251
- // If it is degree 1 then we can deduce the data/aux block immediately.
252
- if (degree == 1) {
253
- long block_nr;
254
-
255
- // Seed with our seed, the chunk index and the degree.
256
- srandom(seed + tmp + degree);
257
-
258
- // This block can be either from ourselves or from our aux blocks.
259
- block_nr = RANDOM(size + RSTRING(aux_blocks)->len);
260
-
261
- //printf(" %d", block_nr);
262
-
263
- // If it is not already known, then set it as known. Otherwise just log about its state.
264
- if (!get_is_known(decode_status, block_nr)) {
265
- set_value(aux_blocks, block_nr, RSTRING(chunk)->ptr[tmp], self, decode_status);
266
- got_new_block = 1;
267
- //printf(" => [%d]=%d", block_nr, RSTRING(chunk)->ptr[tmp]);
268
- } else {
269
- //printf(" [%d]=%d", block_nr, RSTRING(chunk)->ptr[tmp]);
270
- }
271
- } else {
272
- // If it is NOT degree 1, then we go through all the blocks it is xored from
273
- // and see if we have exactly one missing.
274
- long missing_blocks;
275
- long missing_block;
276
- // Create the xor_sum with this block of the chunk.
277
- char xor_sum = RSTRING(chunk)->ptr[tmp];
278
-
279
- for (missing_blocks = tmp2 = degree; tmp2 > 0; tmp2--) {
280
- long block_nr;
281
-
282
- srandom(seed + tmp + tmp2);
283
-
284
- block_nr = RANDOM(size + RSTRING(aux_blocks)->len);
285
-
286
- //printf(" %d", block_nr);
287
-
288
- // If the block we are looking at is known, then we decrement missing_blocks
289
- // and continue with our xor_sum.
290
- if (get_is_known(decode_status, block_nr)) {
291
- missing_blocks--;
292
- xor_sum = xor_sum ^ get_value(aux_blocks, block_nr, self);
293
- //printf("!");
294
- } else {
295
- missing_block = block_nr;
296
- //printf("?");
297
- }
298
- }
299
- // If exactly one block is missing, set is as the final xor_sum.
300
- if (missing_blocks == 1) {
301
- set_value(aux_blocks, missing_block, xor_sum, self, decode_status);
302
- got_new_block = 1;
303
- //printf(" => [%d]=%d", missing_block, xor_sum);
304
- }
305
- }
306
- //printf("\\n");
307
- fflush(NULL);
308
- }
309
-
310
- return got_new_block ? Qtrue : Qfalse;
311
- }
312
- EOC
313
- #
314
- # Get a check block from a message.
315
- #
316
- builder.prefix <<EOC
317
- static char
318
- get_check_block(VALUE self, long seed, long index, VALUE aux_blocks, long (*degree_func)(long))
319
- {
320
- long degree;
321
- int empty_block = 1;
322
- char block;
323
- char found_block;
324
-
325
- // Seed with our seed and the index of this block in the chunk.
326
- srandom(seed + index);
327
-
328
- //printf("creating check block %d:", index);
329
-
330
- // Find [degree] blocks.
331
- for (degree = degree_func(RSTRING(self)->len); degree > 0; degree--) {
332
- long block_number;
333
- char found_block;
334
-
335
- // Seed with our seed, the index in the chunk, and the degree.
336
- srandom(seed + index + degree);
337
-
338
- block_number = RANDOM(RSTRING(self)->len + RSTRING(aux_blocks)->len);
339
- // Depending on whether the block is in our data or our own, debug about it
340
- // and remember what it was.
341
- if (block_number < RSTRING(self)->len) {
342
- found_block = RSTRING(self)->ptr[block_number];
343
- //printf(" d[%d]=%d", block_number, RSTRING(self)->ptr[block_number]);
344
- } else {
345
- found_block = RSTRING(aux_blocks)->ptr[block_number - RSTRING(self)->len];
346
- //printf(" a[%d]=%d", block_number - RSTRING(self)->len, RSTRING(aux_blocks)->ptr[block_number - RSTRING(self)->len]);
347
- }
348
- // Create the block by xoring.
349
- if (empty_block) {
350
- block = found_block;
351
- empty_block = 0;
352
- } else {
353
- block = block ^ found_block;
354
- }
355
- }
356
- //printf(" => %d\\n", block);
357
- return block;
358
- }
359
- EOC
360
- #
361
- # Fill our aux blocks with the proper data.
362
- #
363
- builder.c <<EOC
364
- static void
365
- _fill_aux_blocks(long size, long q)
366
- {
367
- char aux_blocks[size];
368
- char initialized_aux_blocks[size];
369
- long tmp, tmp2;
370
-
371
- for (tmp = 0; tmp < size; tmp++)
372
- initialized_aux_blocks[tmp] = 0;
373
-
374
- srandom(RSTRING(self)->len);
375
-
376
- // For each block of our data.
377
- for (tmp = 0; tmp < RSTRING(self)->len; tmp++) {
378
- //printf("creating aux blocks for d[%d](%d):", tmp, RSTRING(self)->ptr[tmp]);
379
- // Create q blocks.
380
- for (tmp2 = 0; tmp2 < q; tmp2++) {
381
- long aux_block_nr = RANDOM(size);
382
- //printf(" [%d]", aux_block_nr);
383
- // If this aux block is already initialized, just xor to it, otherwise set it.
384
- if (initialized_aux_blocks[aux_block_nr]) {
385
- aux_blocks[aux_block_nr] = aux_blocks[aux_block_nr] ^ RSTRING(self)->ptr[tmp];
386
- } else {
387
- aux_blocks[aux_block_nr] = RSTRING(self)->ptr[tmp];
388
- initialized_aux_blocks[aux_block_nr] = 1;
389
- }
390
- //printf(" => %d", aux_blocks[aux_block_nr]);
391
- }
392
- //printf("\\n");
393
- }
394
- rb_ivar_set(self, rb_intern("@_aux_blocks"), rb_str_new(aux_blocks, size));
395
- }
396
- EOC
397
- #
398
- # Returns a chunk of coded data for a given message.
399
- #
400
- builder.c <<EOC
401
- static VALUE
402
- _get_chunk(long chunk_size, VALUE aux_blocks)
403
- {
404
- unsigned long tmp;
405
- long seed;
406
- struct timeval tmp_timeval;
407
- char chunk[chunk_size];
408
- long (*degree_func)() = get_degree_func(RSTRING(self)->len);
409
-
410
- // Seed with timeofday
411
- gettimeofday(&tmp_timeval, NULL);
412
- srandom(tmp_timeval.tv_usec);
413
- // Get the seed for this chunk.
414
- seed = random();
415
-
416
- ( (long *) chunk) [0] = seed;
417
- ( (long *) chunk) [1] = RSTRING(self)->len;
418
-
419
- // Get the requested number of check blocks and append to this chunk.
420
- for (tmp = 8; tmp < chunk_size; tmp++) {
421
- chunk[tmp] = get_check_block(self, seed, tmp, aux_blocks, degree_func);
422
- }
423
-
424
- return rb_str_new(chunk, chunk_size);
425
- }
426
- EOC
427
- #
428
- # Create the @_aux_hash so that we know what data blocks each aux block represents.
429
- #
430
- builder.c <<EOC
431
- static void
432
- _fill_aux_hash(long q, long size)
433
- {
434
- long tmp, tmp2;
435
- VALUE aux_ary = rb_ary_new();
436
- VALUE set_func = rb_intern("[]=");
437
- VALUE get_func = rb_intern("[]");
438
- VALUE push_func = rb_intern("<<");
439
- VALUE size_func = rb_intern("size");
440
-
441
- srandom(RSTRING(self)->len);
442
-
443
- // Go through all our blocks
444
- for (tmp = 0; tmp < RSTRING(self)->len; tmp++) {
445
- // And for each block q aux blocks.
446
- for (tmp2 = 0; tmp2 < q; tmp2++) {
447
- // And build an Array of the data blocks each aux block represents.
448
- long aux_block_nr = RANDOM(size);
449
- VALUE data_blocks_for_this_aux_block;
450
- if ((data_blocks_for_this_aux_block = rb_funcall(aux_ary, get_func, 1, INT2NUM(aux_block_nr))) == Qnil)
451
- rb_funcall(aux_ary, set_func, 2, INT2NUM(aux_block_nr), (data_blocks_for_this_aux_block = rb_ary_new()));
452
- rb_funcall(data_blocks_for_this_aux_block, push_func, 1, INT2NUM(tmp));
453
- }
454
- }
455
-
456
- // Then go through all our blocks again.
457
- for (tmp = 0; tmp < size; tmp++) {
458
- // Fetch the data blocks for this aux block.
459
- VALUE data_blocks_for_this_aux_block = rb_funcall(aux_ary, get_func, 1, INT2NUM(tmp));
460
- // Count them.
461
- long number_of_data_blocks = NUM2INT(rb_funcall(data_blocks_for_this_aux_block, size_func, 0));
462
- // Create a String that is 4 times the size.
463
- VALUE data_block_string = rb_str_new(0, number_of_data_blocks * 4);
464
- // And go through all q blocks per data block, again.
465
- for (tmp2 = 0; tmp2 < number_of_data_blocks; tmp2++) {
466
- // And make the String contain the data blocks for this aux block, as longs.
467
- rb_str_modify(data_block_string);
468
- ( (long *) RSTRING(data_block_string)->ptr )[tmp2] = (long) NUM2INT(rb_funcall(data_blocks_for_this_aux_block,
469
- get_func,
470
- 1,
471
- INT2NUM(tmp2)));
472
- }
473
- rb_funcall(aux_ary, set_func, 2, INT2NUM(tmp), data_block_string);
474
- }
475
- rb_ivar_set(self, rb_intern("@_aux_hash"), aux_ary);
476
- }
477
- EOC
478
- #
479
- # Try to decode our data and aux blocks until we dont get any more data.
480
- #
481
- builder.c <<EOC
482
- static VALUE
483
- _aux_decode(VALUE decode_status, VALUE aux_blocks, VALUE aux_hash)
484
- {
485
- long tmp, tmp2;
486
- char got_new_block = 0;
487
- VALUE get_func = rb_intern("[]");
488
-
489
- // Go through all aux blocks.
490
- for (tmp = 0; tmp < RSTRING(aux_blocks)->len; tmp++) {
491
- // Only check those that are actually known.
492
- if (get_is_known(decode_status, RSTRING(self)->len + tmp)) {
493
-
494
- //printf("looking at aux block %d(%d): ", tmp, RSTRING(aux_blocks)->ptr[tmp]);
495
-
496
- // Fetch the String defining what data blocks this aux block is made out of.
497
- VALUE data_block_string = rb_funcall(aux_hash, get_func, 1, INT2NUM(tmp));
498
-
499
- // If it is only one data block, then its easy.
500
- if (RSTRING(data_block_string)->len == 4) {
501
- long block_nr = ( (long *) RSTRING(data_block_string)->ptr )[0];
502
-
503
- //printf(" %d", block_nr);
504
-
505
- // If it is not already known, then make it known.
506
- if (!get_is_known(decode_status, block_nr)) {
507
- set_value(aux_blocks, block_nr, RSTRING(aux_blocks)->ptr[tmp], self, decode_status);
508
- got_new_block = 1;
509
- //printf(" => [%d]=%d", block_nr, RSTRING(aux_blocks)->ptr[tmp]);
510
- } else {
511
- //printf(" [%d]=%d", block_nr, RSTRING(aux_blocks)->ptr[tmp]);
512
- }
513
- } else {
514
- // If it is several data blocks, then we just go through them all to see if only
515
- // one of them is unknown.
516
- long missing_blocks;
517
- long missing_block;
518
- // Initialize the xor_sum as this aux_block.
519
- char xor_sum = RSTRING(aux_blocks)->ptr[tmp];
520
-
521
- for (missing_blocks = (tmp2 = ((RSTRING(data_block_string)->len / 4) - 1)) + 1; tmp2 > -1; tmp2--) {
522
- long block_nr = ( (long *) RSTRING(data_block_string)->ptr )[tmp2];
523
-
524
- //printf(" %d", block_nr);
525
-
526
- // If it is a known data block, decrease missing_blocks and append to the xor_sum.
527
- if (get_is_known(decode_status, block_nr)) {
528
- missing_blocks--;
529
- xor_sum = xor_sum ^ get_value(aux_blocks, block_nr, self);
530
- //printf("!(%d)->(%d)", get_value(aux_blocks, block_nr, self), xor_sum);
531
- } else {
532
- missing_block = block_nr;
533
- //printf("?");
534
- }
535
- }
536
- // If only one data block was unknown, then just set it to the built up xor_sum.
537
- if (missing_blocks == 1) {
538
- set_value(aux_blocks, missing_block, xor_sum, self, decode_status);
539
- got_new_block = 1;
540
- //printf(" => [%d]=%d", missing_block, xor_sum);
541
- }
542
- }
543
- }
544
- }
545
- //printf("\\n");
546
- fflush(NULL);
547
-
548
- return got_new_block ? Qtrue : Qfalse;
549
- }
550
- EOC
551
- end
552
-
553
- def oneline_encode(chunk_size)
554
- ensure_aux_blocks
555
- return _get_chunk(chunk_size, @_aux_blocks)
556
- end
557
-
558
- def oneline_decode(chunk)
559
-
560
- needed_size = chunk[4..7].unpack("L").first
561
- if size == 0
562
- concat("\000" * needed_size)
563
- elsif size != needed_size
564
- raise "The argument to oneline_decode must be a chunk produced by oneline_encode from the same source"
565
- end
566
-
567
- @_decode_status ||= "\000" * (((needed_size + get_aux_blocks) / 8) + 1)
568
- @_known_chunks ||= []
569
- @_aux_blocks ||= "\000" * get_aux_blocks
570
-
571
- found_new_block = _decode_chunk(chunk, @_decode_status, @_aux_blocks)
572
- do_decode if found_new_block
573
-
574
- @_known_chunks.unshift(chunk)
575
-
576
- nil while do_decode
577
-
578
- return oneline_done
579
- end
580
-
581
- def oneline_done
582
- return nil unless @_decode_status
583
- return _oneline_done(@_decode_status)
584
- end
585
-
586
- def decode_status
587
- return nil unless @_decode_status
588
- @_decode_status.unpack("b*").first
589
- end
590
-
591
- def decode_level
592
- ones = 0
593
- decode_status[0..(size + @_aux_blocks.size)].split(//).each do |s|
594
- ones += 1 if s == "1"
595
- end
596
- ones.to_f / (size + @_aux_blocks.size).to_f
597
- end
598
-
599
- private
600
-
601
- def get_aux_blocks
602
- (self.size * (get_q * get_e * 0.55)).ceil
603
- end
604
-
605
- def get_q
606
- 0.upto(THRESHOLDS.size - 1) do |n|
607
- if size < THRESHOLDS[n]
608
- return Q[n]
609
- end
610
- end
611
- return Q.last
612
- end
613
-
614
- def do_decode
615
- found_new_block = false
616
- @_known_chunks.each do |known_chunk|
617
- found_new_block = _decode_chunk(known_chunk, @_decode_status, @_aux_blocks) || found_new_block
618
- end
619
- ensure_aux_hash
620
- found_new_block = _aux_decode(@_decode_status, @_aux_blocks, @_aux_hash) || found_new_block
621
- return found_new_block
622
- end
623
-
624
- def get_e
625
- 0.upto(THRESHOLDS.size - 1) do |n|
626
- if size < THRESHOLDS[n]
627
- return E[n]
628
- end
629
- end
630
- return E.last
631
- end
632
-
633
- def ensure_aux_blocks
634
- _fill_aux_blocks(get_aux_blocks, get_q) unless @_aux_blocks
635
- end
636
-
637
- def ensure_aux_hash
638
- _fill_aux_hash(get_q, get_aux_blocks) unless @_aux_hash
639
- end
640
-
641
- end