adhd 0.0.0 → 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Rakefile CHANGED
@@ -14,7 +14,7 @@ begin
14
14
  gem.add_development_dependency "ruby-debug", ">= 0.10.3"
15
15
  gem.add_dependency "sinatra", ">= 0.9.4"
16
16
  gem.add_dependency "couchrest", ">= 0.33"
17
-
17
+ gem.add_dependency "thin", ">= 1.2.4"
18
18
  # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
19
19
  end
20
20
  Jeweler::GemcutterTasks.new
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.0.0
1
+ 0.0.1
@@ -5,13 +5,15 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = %q{adhd}
8
- s.version = "0.0.0"
8
+ s.version = "0.0.1"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["dave.hrycyszyn@headlondon.com"]
12
- s.date = %q{2009-12-18}
12
+ s.date = %q{2009-12-19}
13
+ s.default_executable = %q{adhd}
13
14
  s.description = %q{More to say when something works! Do not bother installing this! }
14
15
  s.email = %q{dave.hrycyszyn@headlondon.com}
16
+ s.executables = ["adhd"]
15
17
  s.extra_rdoc_files = [
16
18
  "LICENSE",
17
19
  "README.rdoc"
@@ -24,9 +26,33 @@ Gem::Specification.new do |s|
24
26
  "Rakefile",
25
27
  "VERSION",
26
28
  "adhd.gemspec",
29
+ "bin/adhd",
27
30
  "doc/adhd.xmi",
28
31
  "lib/adhd.rb",
32
+ "lib/adhd/config.yml",
29
33
  "lib/adhd/models.rb",
34
+ "lib/adhd/node.rb",
35
+ "lib/adhd/reactor.rb",
36
+ "lib/ext/hash_to_openstruct.rb",
37
+ "lib/public/images/img01.jpg",
38
+ "lib/public/images/img02.jpg",
39
+ "lib/public/images/img03.jpg",
40
+ "lib/public/images/img04.jpg",
41
+ "lib/public/images/img05.jpg",
42
+ "lib/public/images/img06.jpg",
43
+ "lib/public/images/img07.jpg",
44
+ "lib/public/images/img08.jpg",
45
+ "lib/public/images/img09.jpg",
46
+ "lib/public/images/img10.gif",
47
+ "lib/public/images/img11.gif",
48
+ "lib/public/images/img12.jpg",
49
+ "lib/public/images/img13.jpg",
50
+ "lib/public/images/img14.jpg",
51
+ "lib/public/images/img15.jpg",
52
+ "lib/public/images/spacer.gif",
53
+ "lib/public/style.css",
54
+ "lib/views/index.erb",
55
+ "lib/views/layout.erb",
30
56
  "models.rb",
31
57
  "test/helper.rb",
32
58
  "test/test_adhd.rb"
@@ -50,17 +76,20 @@ Gem::Specification.new do |s|
50
76
  s.add_development_dependency(%q<ruby-debug>, [">= 0.10.3"])
51
77
  s.add_runtime_dependency(%q<sinatra>, [">= 0.9.4"])
52
78
  s.add_runtime_dependency(%q<couchrest>, [">= 0.33"])
79
+ s.add_runtime_dependency(%q<thin>, [">= 1.2.4"])
53
80
  else
54
81
  s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
55
82
  s.add_dependency(%q<ruby-debug>, [">= 0.10.3"])
56
83
  s.add_dependency(%q<sinatra>, [">= 0.9.4"])
57
84
  s.add_dependency(%q<couchrest>, [">= 0.33"])
85
+ s.add_dependency(%q<thin>, [">= 1.2.4"])
58
86
  end
59
87
  else
60
88
  s.add_dependency(%q<thoughtbot-shoulda>, [">= 0"])
61
89
  s.add_dependency(%q<ruby-debug>, [">= 0.10.3"])
62
90
  s.add_dependency(%q<sinatra>, [">= 0.9.4"])
63
91
  s.add_dependency(%q<couchrest>, [">= 0.33"])
92
+ s.add_dependency(%q<thin>, [">= 1.2.4"])
64
93
  end
65
94
  end
66
95
 
@@ -0,0 +1,33 @@
1
+ #!/usr/bin/env ruby
2
+ require File.dirname(__FILE__) + '/../lib/ext/hash_to_openstruct'
3
+ require File.dirname(__FILE__) + '/../lib/adhd/node'
4
+ require File.dirname(__FILE__) + '/../lib/adhd/reactor'
5
+
6
+ require 'optparse'
7
+ require 'ftools'
8
+ require 'yaml'
9
+ require 'socket'
10
+
11
+ def parse_config(file)
12
+ @config = YAML.load_openstruct(File.read(file))
13
+ end
14
+
15
+ @command = ARGV.shift
16
+
17
+ options = {}
18
+
19
+ opts = OptionParser.new do |opts|
20
+ opts.on("-C", "--config C", "YAML config file") do |n|
21
+ parse_config(n)
22
+ end
23
+ end
24
+
25
+ opts.parse! ARGV
26
+
27
+ @node = Adhd::Node.new(@config)
28
+
29
+ EM.run {
30
+ puts "Starting EventMachine reactor loop..."
31
+ EM.connect @config.node_url, @config.couchdb_server_port, Adhd::DbUpdateReactor, @node
32
+ }
33
+
@@ -5,10 +5,28 @@ require 'erb'
5
5
  require 'ruby-debug'
6
6
  require File.dirname(__FILE__) + '/adhd/models'
7
7
 
8
- node_name = ARGV[0]
9
- node_url = ARGV[1]
10
- buddy_server_url = ARGV[2]
11
- buddy_db = ARGV[3]
8
+ # Start the server for now by cd'ing into the /lib directory and running the
9
+ # following command:
10
+ #
11
+ # (first node):
12
+ # ruby adhd.rb <node_name> <couchdb_server_url>
13
+ #
14
+ # (second or later node)
15
+ # ruby adhd.rb <node_name> <couchdb_server_url> <management_node_url> <management_node_db> -p <port_number>
16
+ #
17
+ # <node_name> is just a string, e.g. "foo".
18
+ # <couchdb_server_url>: the url (including port) for this node's couchdb server
19
+ # instance, e.g, http://192.168.1.104:5984
20
+ # <management_node_url>: the url of the management node where this node should
21
+ # initially replicate from, e.g. http://192.168.1.104:5984
22
+ # <management_node_db>: the couchdb management node database, e.g. "bar_node_db"
23
+ # <port_number>: a port number to run on. If you're running more than one node locally
24
+ # for development purposes you'll need to pick a non-default port higher than 1024.
25
+
26
+ node_name = ARGV[1]
27
+ node_url = ARGV[2]
28
+ buddy_server_url = ARGV[3]
29
+ buddy_db = ARGV[4]
12
30
 
13
31
  NODESERVER = CouchRest.new("#{node_url}")
14
32
  NODESERVER.default_database = "#{node_name}_node_db"
@@ -17,26 +35,86 @@ node_db = CouchRest::Database.new(NODESERVER, "#{node_name}_node_db")
17
35
  # sync the db with our buddy
18
36
  if buddy_server_url && buddy_db
19
37
  buddy_server = CouchRest.new("#{buddy_server_url}")
20
- buddy_db = CouchRest::Database.new(buddy_server, buddy_db)
38
+ buddy_db = CouchRest::Database.new(buddy_server, buddy_db + "_node_db")
21
39
  node_db.replicate_from(buddy_db)
22
40
  end
23
41
 
24
- node = Node.by_name(:key => node_name).first
42
+ # Retrieve our own node by our name
43
+ # If there are other nodes with the name kill their records!
44
+ node_candidates = Node.by_name(:key => node_name)
45
+ node = node_candidates.pop
25
46
  node = Node.new if node.nil?
47
+ node_candidates.each do |other_me|
48
+ other_me.destroy # destroy other records
49
+ end
26
50
 
51
+ # Update our very own record
27
52
  node.name = node_name
28
53
  node.url = node_url
54
+ node.status = "RUNNING"
29
55
  node.save
30
56
 
31
57
  # We check if we are the first node. If we are the first node, we set ourself up
32
- # as the management node. If not, we find out where the management node is and
33
- # we replicate to the administrative node.
34
- if management_node = Node.by_is_management.last
35
- management_node_server = CouchRest.new(management_node.url)
36
- management_node_db = CouchRest::Database.new(management_node_server, management_node.name + "_node_db")
37
- node_db.replicate_to(management_node_db)
38
- else
58
+ # as the management node.
59
+ all_nodes = Node.by_name()
60
+ if all_nodes.length == 1
61
+ # puts "Setup #{node.name} as management node"
39
62
  node.is_management = 3
40
63
  node.save
41
64
  end
42
65
 
66
+ # Lets build a nice NodeDB
67
+ ndb = NodeDB.new(node)
68
+
69
+ # Lets build a nice ShardDB
70
+ srdb = ShardRangeDB.new(ndb)
71
+
72
+ # If there are no shards make a few, if we are managers
73
+ #puts "Create new ranges?"
74
+ #puts "How many shards: #{ShardRange.by_range_start.length}"
75
+ #puts "in #{ShardRange::SHARDSERVER.default_database}"
76
+ if ShardRange.by_range_start.length == 0 && node.is_management
77
+ puts "Creating new ranges"
78
+ srdb.build_shards(100)
79
+ end
80
+
81
+ # Polulate the shards with some nodes at random
82
+ node_names = []
83
+ all_nodes.each do |anode|
84
+ node_names << anode.name
85
+ end
86
+
87
+ ShardRange.by_range_start.each do |s|
88
+ if !s.node_list or s.node_list.length == 0
89
+ node_names.shuffle!
90
+ s.node_list = node_names[0..2]
91
+ s.master_node = node_names[0]
92
+ s.save
93
+ end
94
+
95
+ end
96
+ # Sync all the node databases
97
+
98
+ ndb.sync # SYNC
99
+ srdb.sync # SYNC
100
+
101
+ srdb.get_content_shards.each do |content_shard_db|
102
+ content_shard_db.sync
103
+ end
104
+
105
+ get "/" do
106
+ @all_nodes = Node.by_name
107
+ erb :index
108
+ end
109
+
110
+ get "/sync" do
111
+ # Sync the node database
112
+ ndb.sync
113
+ # Sync the shard database
114
+ srdb.sync
115
+
116
+ srdb.get_content_shards.each do |content_shard_db|
117
+ content_shard_db.sync
118
+ end
119
+ end
120
+
@@ -0,0 +1,7 @@
1
+ ---
2
+ node_name: superfoo
3
+ node_url: "192.168.1.104"
4
+ couchdb_server_port: 5984
5
+ buddy_server_url: "http://192.168.1.104:5984" # It is a good idea if this is a management node
6
+ buddy_server_name: foo
7
+
@@ -1,3 +1,59 @@
1
+ # Key Restrictions ok internal_IDs: must only contain [a-z0-9-]
2
+
3
+ class Array
4
+ def shuffle!
5
+ size.downto(1) { |n| push delete_at(rand(n)) }
6
+ self
7
+ end
8
+ end
9
+
10
+ class NodeDB
11
+
12
+ attr_accessor :local_node_db, :our_node
13
+
14
+ def initialize(our_nodev)
15
+ @our_node = our_nodev
16
+
17
+ # Get the address of the CDB from the node
18
+ @local_node_db = our_nodev.get_node_db
19
+ end
20
+
21
+ def sync
22
+ # We replicate our state to the management node(s)
23
+ management_nodes = Node.by_is_management.reverse
24
+ # NOTE: randomize the order for load balancing here
25
+
26
+ # NOTE2: How to build skynet (TODO)
27
+ # -------------------
28
+ # If length of management is zero, then chose 3 different random
29
+ # nodes at each sync, and sync with them in node_name order.
30
+ # This guarantees that any updates on nodes are communicated in
31
+ # O(log N) ephocs, at the cost of O(3 * N) connections per epoch.
32
+ # It also guarantees any new management servers are discovered in
33
+ # this O(log N) time, creating "jelly fish" or "partition proof"
34
+ # availability.
35
+
36
+ management_nodes.each do |mng_node|
37
+ remote_db = mng_node.get_node_db
38
+ if !(mng_node.name == our_node.name)
39
+ begin
40
+ puts "Sync NodeDB with #{mng_node.name}"
41
+ local_node_db.replicate_from(remote_db)
42
+ # TODO: Manage conflicts here
43
+ local_node_db.replicate_to(remote_db)
44
+ break if !our_node.is_management # Only need to contact one node
45
+ rescue
46
+ puts "Could not connect to DB node #{mng_node.name}"
47
+ # TODO: change status or chose another management server
48
+ mng_node.status = "UNAVAILABLE"
49
+ mng_node.save
50
+ end
51
+ end
52
+ end
53
+ end
54
+
55
+ end
56
+
1
57
  class Node < CouchRest::ExtendedDocument
2
58
  NODESERVER = CouchRest.new("#{ARGV[1]}")
3
59
  NODESERVER.default_database = "#{ARGV[0]}_node_db"
@@ -15,5 +71,318 @@ class Node < CouchRest::ExtendedDocument
15
71
 
16
72
  view_by :name
17
73
  view_by :is_management
74
+
75
+ def get_node_db
76
+ server = CouchRest.new("#{url}")
77
+ server.database!("#{name}_node_db")
78
+ end
79
+
80
+ def get_shard_db
81
+ server = CouchRest.new("#{url}")
82
+ server.database!("#{name}_shard_db")
83
+ end
84
+
85
+ def get_content_db(shard_db_name)
86
+ server = CouchRest.new("#{url}")
87
+ server.database!("#{name}_#{shard_db_name}_content_db")
88
+ end
89
+ end
90
+
91
+ class ShardRangeDB
92
+
93
+ attr_accessor :nodes, :local_shard_db, :our_node
94
+
95
+ def initialize(nodesv)
96
+ @nodes = nodesv
97
+
98
+ # Automatically get our shard_db address from our own node name
99
+ @our_node = nodesv.our_node
100
+ @local_shard_db = nodesv.our_node.get_shard_db
101
+ end
102
+
103
+ def sync
104
+ # We replicate our state from the management node(s)
105
+ # We never push content if we are only storage
106
+ management_nodes = Node.by_is_management.reverse
107
+ # NOTE: randomize the order for load balancing here
108
+
109
+
110
+ management_nodes.each do |mng_node|
111
+ remote_db = mng_node.get_shard_db
112
+ if !(mng_node.name == our_node.name)
113
+ begin
114
+ puts "Sync ShardRange DB pull from #{mng_node.name}"
115
+ local_shard_db.replicate_from(remote_db)
116
+ # TODO: Manage conflicts here
117
+ if our_node.is_management
118
+ # Push any changes to other management nodes
119
+ puts "Sync ShardRange DB pushto #{mng_node.name}"
120
+ local_shard_db.replicate_to(remote_db)
121
+ else
122
+ break # sync with only one management server
123
+ end
124
+ rescue
125
+ puts "Could not connect to DB node #{mng_node.name}"
126
+ # TODO: change status or chose another management server
127
+ mng_node.status = "UNAVAILABLE"
128
+ mng_node.save
129
+ end
130
+ end
131
+ end
132
+ end
133
+
134
+ def build_shards(number)
135
+ # Make a large list of possible id boundaries
136
+ characters = []
137
+ ("0".."9").each do |c|
138
+ characters << c
139
+ end
140
+ ("a".."z").each do |c|
141
+ characters << c
142
+ end
143
+
144
+ # Generate 36 x 36 keys to choose boundaries from
145
+ all_keys = []
146
+ characters.each do |c1|
147
+ characters.each do |c2|
148
+ all_keys << (c1+c2)
149
+ end
150
+ end
151
+
152
+ # Now chose our boundaries
153
+ num_range_keys = all_keys.length
154
+ approx_shard_size = (num_range_keys * 1.0) / number
155
+
156
+ shard_starts = []
157
+ (0...number).each do |n|
158
+ shard_starts << (all_keys[(n * approx_shard_size).floor])
159
+ end
160
+
161
+ shard_ends = shard_starts.clone
162
+ shard_ends << ("z" * 100)
163
+ shard_ends.delete_at(0)
164
+
165
+ # Finally build them!
166
+ puts "Build Shards"
167
+ (0...number).each do |n|
168
+ puts "Shard #{n}: from #{shard_starts[n]} to #{shard_ends[n]}"
169
+ shard_name = "sh_#{shard_starts[n]}_to_#{shard_ends[n]}"
170
+ sr = ShardRange.new
171
+ sr.range_start = shard_starts[n]
172
+ sr.range_end = shard_ends[n]
173
+ sr.shard_db_name = shard_name
174
+ sr.save
175
+ end
176
+ end
177
+
178
+ def get_shard(internal_id)
179
+ # Finds the list of shards within which this ID lives
180
+ all_shards = ShardRange.by_range_start
181
+ selected_shards = []
182
+ all_shards.each do |a_shard| # TODO: linear search is inefficient -- create a view
183
+ if a_shard.range_start <= internal_id && a_shard.range_end > internal_id
184
+ selected_shards << a_shard
185
+ end
186
+ end
187
+ selected_shards
188
+ end
189
+
190
+ def get_content_shards
191
+ # Return the content_shards of our node
192
+ content_shards = []
193
+ ShardRange.by_node(:key => "node1").each do |s|
194
+
195
+ # Build a content shard object
196
+ cs = ContentShard.new
197
+ cs.our_node = our_node
198
+ cs.this_shard = s
199
+ cs.nodes = nodes
200
+ cs.this_shard_db = our_node.get_content_db(s.shard_db_name)
201
+
202
+ # add it to the list
203
+ content_shards << cs
204
+ end
205
+ content_shards
206
+ end
207
+
208
+ def write_doc_directly(content_doc)
209
+ # Write a document directly to a nodes content repository
210
+ doc_shard = get_shard(content_doc.internal_id).first
211
+ doc_shard.get_nodes.each do |node|
212
+ # Try to write the doc to this node
213
+ begin
214
+ remote_ndb = NodeDB.new(node)
215
+ remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
216
+ remote_content_shard.this_shard_db.save_doc(content_doc)
217
+ break
218
+ rescue
219
+ puts "Could not put doc in node #{node.name}"
220
+ # TODO: change status or chose another management server
221
+ node.status = "UNAVAILABLE"
222
+ node.save
223
+
224
+ end
225
+ end
226
+
227
+ end
228
+
229
+ def get_doc_directly(internal_id)
230
+ # Write a document directly to a nodes content repository
231
+ doc_shard = get_shard(internal_id).first
232
+
233
+ # TODO: Randomize the order of nodes for load balancing in retrieval!
234
+ docx = []
235
+ doc_shard.get_nodes.each do |node|
236
+ # Try to write the doc to this node
237
+ begin
238
+ remote_ndb = NodeDB.new(node)
239
+ remote_content_shard = ContentShard.new(remote_ndb, doc_shard)
240
+
241
+ docx = ContentDoc.by_internal_id(:key => internal_id, :database => remote_content_shard.this_shard_db)
242
+ if docx.length > 0
243
+ break
244
+ end
245
+ rescue
246
+ puts "Could not put doc in node #{node.name}"
247
+ # TODO: change status or chose another management server
248
+ node.status = "UNAVAILABLE"
249
+ node.save
250
+ end
251
+ end
252
+ docx
253
+ end
254
+
255
+ end
256
+
257
+ class ShardRange < CouchRest::ExtendedDocument
258
+ SHARDSERVER = CouchRest.new("#{ARGV[1]}")
259
+ SHARDSERVER.default_database = "#{ARGV[0]}_shard_db"
260
+
261
+ use_database SHARDSERVER.default_database
262
+
263
+ property :range_start
264
+ property :range_end
265
+ property :node_list
266
+ property :master_node
267
+ property :shard_db_name
268
+
269
+ view_by :range_start
270
+
271
+ # View "node" - given a node returns the shards watched
272
+ # How to use this new
273
+ #
274
+ # puts "Which Shards does 'node1' watch?"
275
+ # ShardRange.by_node(:key => "node1").each do |s|
276
+ # puts "Shard: #{s.shard_db_name}"
277
+ # end
278
+
279
+
280
+ view_by :node,
281
+ :map =>
282
+ "function(doc) {
283
+ if (doc['couchrest-type'] == 'ShardRange' && doc.node_list) {
284
+ doc.node_list.forEach(function(node){
285
+ emit(node, 1);
286
+ });
287
+ }
288
+ }"
289
+
290
+ def get_nodes
291
+ # Return all nodes, with the master being first
292
+ all_nodes = node_list
293
+ all_nodes.delete(master_node)
294
+ all_nodes = [master_node] + all_nodes
295
+ allnodes
296
+ end
297
+
18
298
  end
19
299
 
300
+
301
+ class ContentShard
302
+ attr_accessor :nodes, :this_shard, :our_node, :this_shard_db
303
+
304
+ def initialize(nodesv, this_shardv)
305
+ @nodes = nodesv
306
+ @this_shard = this_shardv
307
+
308
+ # Work out the rest
309
+ @our_node = nodesv.our_node
310
+ @this_shard_db = nodesv.our_node.get_content_db(this_shardv.shard_db_name)
311
+ end
312
+
313
+ def in_shard?(internal_id)
314
+ internal_id >= this_shard.range_start && internal_id < this_shard.range_end
315
+ end
316
+
317
+ def write_doc(content_doc)
318
+ # Write a content document to this shard
319
+ # Make sure it is in this shard
320
+ if in_shard? content_doc.internal_id
321
+ this_shard_db.save_doc(content_doc)
322
+ end
323
+ end
324
+
325
+ def sync
326
+ # A Shard only pushes with the master of the shard
327
+ # or the node with the highest is_storage value alive
328
+ # Shard masters ensure changes are pushed to all
329
+
330
+ # NOTE: This method needs serious refactoring
331
+
332
+ # Are we the shard master?
333
+ am_master = false
334
+ if our_node.name == this_shard.master_node
335
+ am_master = true
336
+ end
337
+
338
+ if !am_master
339
+ begin
340
+ master_node = Nodes.by_name(this_shard.master_node).first
341
+ remotedb = MASTER_node.get_content_db(this_shard.shard_db_name)
342
+ this_shard_db.replicate_to(remote_db)
343
+ return # We sync-ed so job is done
344
+ rescue
345
+ # We flag the master as unavailable
346
+ if remote_node
347
+ master_node.status = "UNAVAILABLE"
348
+ master_node.save
349
+ end
350
+ end
351
+ end
352
+
353
+ # Either we are the master or the master has failed -- we replicate with
354
+ # all nodes or the first available aside us and master
355
+ this_shard.node_list.each do |node_name|
356
+ if !(our_node.name == node_name) && !(this_shard.master_node == node_name)
357
+ begin
358
+ # Push all changes to the other nodes
359
+ remote_node = Nodes.by_name(node_name).first
360
+ remotedb = remote_node.get_content_db(this_shard.shard_db_name)
361
+ this_shard_db.replicate_to(remote_db)
362
+ break if !am_master
363
+ rescue
364
+ # Make sure that the node exist in the DB and flag it as unresponsive
365
+ if remote_node
366
+ remote_node.status = "UNAVAILABLE"
367
+ remote_node.save
368
+ end
369
+ end
370
+ end
371
+
372
+ end
373
+ end
374
+ end
375
+
376
+ class ContentDoc < CouchRest::ExtendedDocument
377
+ # NOTE: NO DEFAULT DATABASE IN THE OBJECT -- WE WILL BE STORING A LOT OF
378
+ # DATABASES OF THIS TYPE.
379
+
380
+ property :internal_id
381
+ property :size_bytes
382
+ property :filenane
383
+ property :mime_type
384
+
385
+ view_by :internal_id
386
+
387
+ # A special attachment "File" is expected to exist
388
+ end