distribustream 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,122 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ # All rights reserved. See COPYING for permissions.
4
+ #
5
+ # This source file is distributed as part of the
6
+ # DistribuStream file transfer system.
7
+ #
8
+ # See http://distribustream.rubyforge.org/
9
+ #++
10
+
11
+ #Provides functions used for initialization by both the client and server
12
+
13
+ require 'optparse'
14
+ require 'logger'
15
+
16
+ require File.dirname(__FILE__) + '/protocol'
17
+
18
+ STDOUT.sync=true
19
+ STDERR.sync=true
20
+
21
+ @@log=Logger.new(STDOUT)
22
+ @@log.datetime_format=""
23
+
24
+ CONFIG_TYPES = {
25
+ :host => :string,
26
+ :vhost => :string,
27
+ :port => :int,
28
+ :listen_port => :int,
29
+ :file_root => :string,
30
+ :quiet => :bool,
31
+ :chunk_size => :int,
32
+ :request_url => :string
33
+ }
34
+
35
+ #prints banner and loads config file
36
+ def common_init(program_name, config = nil)
37
+ @@config = config || {
38
+ :host => '0.0.0.0',
39
+ :port => 6086, #server port
40
+ :listen_port => 8000, #client listen port
41
+ :file_root => '.',
42
+ :chunk_size => 5000,
43
+ :quiet => true
44
+ }
45
+
46
+ config_filename=nil
47
+
48
+ unless config
49
+ OptionParser.new do |opts|
50
+ opts.banner = "Usage: #{program_name} [options]"
51
+ opts.on("--config CONFIGFILE", "Load specified config file.") do |c|
52
+ config_filename=c
53
+ end
54
+ opts.on("--help", "Prints this usage info.") do
55
+ puts opts
56
+ exit
57
+ end
58
+ end.parse!
59
+ end
60
+
61
+ puts "#{program_name} starting. Run '#{program_name} --help' for more info."
62
+
63
+ load_config_file(config_filename) unless config
64
+
65
+ begin
66
+ @@config[:file_root]=File.expand_path(@@config[:file_root])
67
+ rescue
68
+ puts "Invalid path specified for file_root"
69
+ return
70
+ end
71
+
72
+ puts "@@config=#{@@config.inspect}"
73
+ validate_config_options
74
+ handle_config_options
75
+ end
76
+
77
+ #loads a config file specified by config_filename
78
+ def load_config_file(config_filename)
79
+ if config_filename.nil?
80
+ puts "No config file specified. Using defaults."
81
+ return
82
+ end
83
+
84
+ confstr=File.read(config_filename) rescue nil
85
+ if confstr.nil?
86
+ puts "Unable to open config file: #{config_filename}"
87
+ exit
88
+ end
89
+
90
+ begin
91
+ new_config = YAML.load confstr
92
+ @@config.merge!(new_config)
93
+ @@config[:vhost] ||= @@config[:host] # Use host as vhost unless specified
94
+ rescue Exception => e
95
+ puts "Error parsing config file: #{config_filename}"
96
+ puts e
97
+ exit
98
+ end
99
+
100
+ puts "Loaded config file: #{config_filename}"
101
+ end
102
+
103
+ #make sure all the config options are of the right type
104
+ def validate_config_options
105
+ @@config.each do |key,val|
106
+ type=CONFIG_TYPES[key]
107
+ if type.nil?
108
+ puts "Unknown parameter: #{key}"
109
+ exit
110
+ end
111
+
112
+ unless PDTP::Protocol.obj_matches_type?(val,type)
113
+ puts "Parameter: #{key} is not of type: #{type}"
114
+ exit
115
+ end
116
+ end
117
+ end
118
+
119
+ #responds to config options that are used by both client and server
120
+ def handle_config_options
121
+ @@log.level=Logger::INFO if @@config[:quiet]
122
+ end
@@ -0,0 +1,69 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ # All rights reserved. See COPYING for permissions.
4
+ #
5
+ # This source file is distributed as part of the
6
+ # DistribuStream file transfer system.
7
+ #
8
+ # See http://distribustream.rubyforge.org/
9
+ #++
10
+
11
+ module PDTP
12
+ #provides information about a single file on the network
13
+ class FileInfo
14
+ attr_accessor :file_size, :base_chunk_size, :streaming
15
+
16
+ #number of chunks in the file
17
+ def num_chunks
18
+ return 0 if @file_size==0
19
+ (@file_size - 1) / @base_chunk_size + 1
20
+ end
21
+
22
+ #size of the specified chunk
23
+ def chunk_size(chunkid)
24
+ raise "Invalid chunkid #{chunkid}" if chunkid<0 or chunkid>=num_chunks
25
+ chunkid == num_chunks - 1 ? @file_size - @base_chunk_size * chunkid : @base_chunk_size
26
+ end
27
+
28
+ #range of bytes taken up by this chunk in the entire file
29
+ def chunk_range(chunkid)
30
+ start_byte = chunkid * @base_chunk_size
31
+ end_byte = start_byte + chunk_size(chunkid) - 1
32
+ start_byte..end_byte
33
+ end
34
+
35
+ #returns the chunkid that contains the requested byte offset
36
+ def chunk_from_offset(offset)
37
+ raise "Invalid offset #{offset}" if offset < 0 or offset >= @file_size
38
+ offset / @base_chunk_size
39
+ end
40
+
41
+ #takes a byte_range in the file and returns an equivalent chunk range
42
+ #if exclude_partial is true, chunks that are not completely covered by the byte range are left out
43
+ def chunk_range_from_byte_range(byte_range,exclude_partial=true)
44
+ min=chunk_from_offset(byte_range.first)
45
+ min+=1 if exclude_partial and byte_range.first > min*@base_chunk_size
46
+
47
+ max_byte=byte_range.last
48
+ max_byte=@file_size-1 if max_byte==-1 or max_byte>=@file_size
49
+ max=chunk_from_offset(max_byte)
50
+ max-=1 if exclude_partial and max_byte<chunk_range(max).last
51
+ min..max
52
+ end
53
+
54
+ #returns a string containing the data for this chunk
55
+ #range specifies a range of bytes local to this chunk
56
+ #implemented in Client and Server file services
57
+ def chunk_data(chunkid,range=nil)
58
+ end
59
+ end
60
+
61
+ # base class for ClientFileService and ServerFileService.
62
+ # provides shared functionality
63
+
64
+ class FileService
65
+ #returns a FileInfo class associated with the url, or nil if the file isnt known
66
+ def get_info(url)
67
+ end
68
+ end
69
+ end
@@ -0,0 +1,91 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ # All rights reserved. See COPYING for permissions.
4
+ #
5
+ # This source file is distributed as part of the
6
+ # DistribuStream file transfer system.
7
+ #
8
+ # See http://distribustream.rubyforge.org/
9
+ #++
10
+
11
+ require File.dirname(__FILE__) + '/file_service'
12
+
13
+ describe "A FileInfo with chunk_size=1" do
14
+ before(:each) do
15
+ @fi=PDTP::FileInfo.new
16
+ @fi.file_size=5
17
+ @fi.base_chunk_size=1
18
+ end
19
+
20
+ it "chunk_size works" do
21
+ @fi.chunk_size(0).should == 1
22
+ @fi.chunk_size(3).should == 1
23
+ @fi.chunk_size(4).should == 1
24
+
25
+ proc{ @fi.chunk_size(-1)}.should raise_error
26
+ proc{ @fi.chunk_size(5)}.should raise_error
27
+ end
28
+
29
+ it "num_chunks works" do
30
+ @fi.num_chunks.should == 5
31
+ end
32
+
33
+ it "chunk_from_offset works" do
34
+ @fi.chunk_from_offset(0).should == 0
35
+ @fi.chunk_from_offset(4).should == 4
36
+ proc{@fi.chunk_from_offset(5)}.should raise_error
37
+ end
38
+
39
+ it "chunk_range_from_byte_range works" do
40
+ @fi.chunk_range_from_byte_range(0..4,false).should == (0..4)
41
+ @fi.chunk_range_from_byte_range(0..4,true).should == (0..4)
42
+ proc{@fi.chunk_range_from_byte_range(-1..3,true)}.should raise_error
43
+ end
44
+
45
+ end
46
+
47
+ describe "A FileInfo with chunk_size=256 and file_size=768" do
48
+ before(:each) do
49
+ @fi=PDTP::FileInfo.new
50
+ @fi.base_chunk_size=256
51
+ @fi.file_size=768
52
+ end
53
+
54
+ it "chunk_size works" do
55
+ @fi.chunk_size(0).should == 256
56
+ @fi.chunk_size(2).should == 256
57
+ proc{@fi.chunk_size(3)}.should raise_error
58
+ end
59
+
60
+ it "num_chunks works" do
61
+ @fi.num_chunks.should == 3
62
+ end
63
+
64
+ it "chunk_from_offset works" do
65
+ @fi.chunk_from_offset(256).should == 1
66
+ @fi.chunk_from_offset(255).should == 0
67
+ end
68
+
69
+ it "chunk_range_from_byte_range works" do
70
+ @fi.chunk_range_from_byte_range(256..511,true).should == (1..1)
71
+ @fi.chunk_range_from_byte_range(256..511,false).should == (1..1)
72
+ @fi.chunk_range_from_byte_range(255..512,true).should == (1..1)
73
+ @fi.chunk_range_from_byte_range(255..512,false).should == (0..2)
74
+ end
75
+ end
76
+
77
+ describe "A FileInfo with chunk_size=256 and file_size=255" do
78
+ before(:each) do
79
+ @fi=PDTP::FileInfo.new
80
+ @fi.base_chunk_size=256
81
+ @fi.file_size=255
82
+ end
83
+
84
+ it "num_chunks works" do
85
+ @fi.num_chunks.should ==1
86
+ end
87
+
88
+ it "chunk_from_offset works" do
89
+ @fi.chunk_from_offset(254).should == 0
90
+ end
91
+ end
@@ -0,0 +1,346 @@
1
+ #--
2
+ # Copyright (C) 2006-07 ClickCaster, Inc. (info@clickcaster.com)
3
+ # All rights reserved. See COPYING for permissions.
4
+ #
5
+ # This source file is distributed as part of the
6
+ # DistribuStream file transfer system.
7
+ #
8
+ # See http://distribustream.rubyforge.org/
9
+ #++
10
+
11
+ require 'rubygems'
12
+ require 'eventmachine'
13
+ require 'thread'
14
+ require 'uri'
15
+ require 'ipaddr'
16
+
17
+ begin
18
+ require 'fjson'
19
+ rescue LoadError
20
+ require 'json'
21
+ end
22
+
23
+ module PDTP
24
+ PROTOCOL_DEBUG=true
25
+
26
+ class ProtocolError < Exception
27
+ end
28
+
29
+ class ProtocolWarn < Exception
30
+ end
31
+
32
+ # EventMachine handler class for the PDTP protocol
33
+ class Protocol < EventMachine::Protocols::LineAndTextProtocol
34
+ @@num_connections = 0
35
+ @@listener = nil
36
+ @@message_params = nil
37
+ @connection_open = false
38
+
39
+ def connection_open?
40
+ @connection_open
41
+ end
42
+
43
+ #sets the listener class (Server or Client)
44
+ def self.listener=(listener)
45
+ @@listener = listener
46
+ end
47
+
48
+ def initialize(*args)
49
+ user_data = nil
50
+ @mutex = Mutex.new
51
+ super
52
+ end
53
+
54
+ #called by EventMachine after a connection has been established
55
+ def post_init
56
+ # a cache of the peer info because eventmachine seems to drop it before we want
57
+ peername = get_peername
58
+ if peername.nil?
59
+ @cached_peer_info = ["<Peername nil!!!>", 91119] if peername.nil?
60
+ else
61
+ port, addr = Socket.unpack_sockaddr_in(peername)
62
+ @cached_peer_info = [addr.to_s, port.to_i]
63
+ end
64
+
65
+ @@num_connections += 1
66
+ @connection_open = true
67
+ @@listener.connection_created(self) if @@listener.respond_to?(:connection_created)
68
+ end
69
+
70
+ attr_accessor :user_data #users of this class may store arbitrary data here
71
+
72
+ #close a connection, but first send the specified error message
73
+ def error_close_connection(error)
74
+ if PROTOCOL_DEBUG
75
+ send_message :protocol_error, :message => msg
76
+ close_connection(true) # close after writing
77
+ else
78
+ close_connection
79
+ end
80
+ end
81
+
82
+ #override this in a child class to handle messages
83
+ def receive_message(command, message)
84
+ @@listener.dispatch_message command, message, self
85
+ end
86
+
87
+ #debug routine: returns id of remote peer on this connection
88
+ def remote_peer_id
89
+ ret = user_data.client_id rescue nil
90
+ ret || 'NOID'
91
+ end
92
+
93
+ #called for each line of text received over the wire
94
+ #parses the JSON message and dispatches the message
95
+ def receive_line line
96
+ begin
97
+ line.chomp!
98
+ @@log.debug "(#{remote_peer_id}) recv: " + line
99
+ message = JSON.parse(line) rescue nil
100
+ raise ProtocolError.new("JSON couldn't parse: #{line}") if message.nil?
101
+ Protocol.validate_message message
102
+
103
+ command, options = message
104
+ hash_to_range command, options
105
+ receive_message command, options
106
+ rescue ProtocolError => e
107
+ @@log.warn "(#{remote_peer_id}) PROTOCOL ERROR: #{e.to_s}"
108
+ @@log.debug e.backtrace.join("\n")
109
+ error_close_connection e.to_s
110
+ rescue ProtocolWarn => e
111
+ send_message :protocol_warn, :message => e.to_s
112
+ rescue Exception => e
113
+ puts "(#{remote_peer_id}) UNKNOWN EXCEPTION #{e.to_s}"
114
+ puts e.backtrace.join("\n")
115
+ end
116
+ end
117
+
118
+ RANGENAMES = %w{chunk_range range byte_range}
119
+
120
+ #converts Ruby Range classes in the message to PDTP protocol hashes with min and max
121
+ # 0..-1 => nil (entire file)
122
+ # 10..-1 => {"min"=>10} (contents of file >= 10)
123
+ def range_to_hash(message)
124
+ message.each do |key,value|
125
+ if value.class==Range
126
+ if value==(0..-1)
127
+ message.delete(key)
128
+ elsif value.last==-1
129
+ message[key]={"min"=>value.first}
130
+ else
131
+ message[key]={"min"=>value.first,"max"=>value.last}
132
+ end
133
+ end
134
+ end
135
+ end
136
+
137
+ #converts a PDTP protocol min and max hash to a Ruby Range class
138
+ def hash_to_range(command, message)
139
+ key="range"
140
+ auto_types=["provide","request"] #these types assume a range if it isnt specified
141
+ auto_types.each do |type|
142
+ if command == type and message[key].nil?
143
+ message[key]={} # assume entire file if not specified
144
+ end
145
+ end
146
+
147
+ if message[key]
148
+ raise if message[key].class!=Hash
149
+ min=message[key]["min"]
150
+ max=message[key]["max"]
151
+ message[key]= (min ? min : 0)..(max ? max : -1)
152
+ end
153
+ end
154
+
155
+ #sends a message, in the internal Hash format, over the wire
156
+ def send_message(command, opts = {})
157
+ #message = opts.merge(:type => command.to_s)
158
+
159
+ # Stringify all option keys
160
+ opts = opts.map { |k,v| [k.to_s, v] }.inject({}) { |h,(k,v)| h[k] = v; h }
161
+
162
+ # Convert all Ruby ranges to JSON objects representing them
163
+ range_to_hash opts
164
+
165
+ # Message format is a JSON array with the command (string) as the first entry
166
+ # Second entry is an options hash/object
167
+ message = [command.to_s, opts]
168
+
169
+ @mutex.synchronize do
170
+ outstr = JSON.unparse(message) + "\n"
171
+ @@log.debug "(#{remote_peer_id}) send: #{outstr.chomp}"
172
+ send_data outstr
173
+ end
174
+ end
175
+
176
+ #called by EventMachine when a connection is closed
177
+ def unbind
178
+ @@num_connections -= 1
179
+ @@listener.connection_destroyed(self) if @@listener.respond_to?(:connection_destroyed)
180
+ @connection_open = false
181
+ end
182
+
183
+ def self.print_info
184
+ puts "num_connections=#{@@num_connections}"
185
+ end
186
+
187
+ #returns the ip address and port in an array [ip, port]
188
+ def get_peer_info
189
+ @cached_peer_info
190
+ end
191
+
192
+ def to_s
193
+ addr,port = get_peer_info
194
+ "#{addr}:#{port}"
195
+ end
196
+
197
+ #makes sure that the message is valid.
198
+ #if not, throws a ProtocolError
199
+ def self.validate_message(message)
200
+ raise ProtocolError.new("Message is not a JSON array") unless message.is_a? Array
201
+ command, options = message
202
+
203
+ @@message_params ||= define_message_params
204
+
205
+ params = @@message_params[command] rescue nil
206
+ raise ProtocolError.new("Invalid message type: #{command}") if params.nil?
207
+
208
+ params.each do |name,type|
209
+ if type.class == Optional
210
+ next if options[name].nil? #dont worry about it if they dont have this param
211
+ type = type.type #grab the real type from within the optional class
212
+ end
213
+
214
+ raise ProtocolError.new("required parameter: '#{name}' missing for message: '#{command}'") if options[name].nil?
215
+ unless obj_matches_type?(options[name], type)
216
+ raise ProtocolError.new("parameter: '#{name}' val='#{options[name]}' is not of type: '#{type}' for message: '#{command}' ")
217
+ end
218
+ end
219
+ end
220
+
221
+ # an optional field of the specified type
222
+ class Optional
223
+ attr_accessor :type
224
+ def initialize(type)
225
+ @type=type
226
+ end
227
+ end
228
+
229
+ #returns whether or not a given ruby object matches the specified type
230
+ #available types:
231
+ # :url, :range, :ip, :int, :bool, :string
232
+ def self.obj_matches_type?(obj,type)
233
+ case type
234
+ when :url then obj.class == String
235
+ when :range then obj.class == Range or obj.class == Hash
236
+ when :int then obj.class == Fixnum
237
+ when :bool then obj == true or obj == false
238
+ when :string then obj.class == String
239
+ when :ip
240
+ ip = IPAddr.new(obj) rescue nil
241
+ !ip.nil?
242
+ else
243
+ raise "Invalid type specified: #{type}"
244
+ end
245
+ end
246
+
247
+ #this function defines the required fields for each message
248
+ def self.define_message_params
249
+ mp = {}
250
+
251
+ #must be the first message the client sends
252
+ mp["client_info"]={
253
+ "client_id"=>:string,
254
+ "listen_port"=>:int
255
+ }
256
+
257
+ mp["ask_info"]={
258
+ "url"=>:url
259
+ }
260
+
261
+ mp["tell_info"]={
262
+ "url"=>:url,
263
+ "size"=>Optional.new(:int),
264
+ "chunk_size"=>Optional.new(:int),
265
+ "streaming"=>Optional.new(:bool)
266
+ }
267
+
268
+ mp["ask_verify"]={
269
+ "peer"=>:ip,
270
+ "url"=>:url,
271
+ "range"=>:range,
272
+ "peer_id"=>:string
273
+ }
274
+
275
+ mp["tell_verify"]={
276
+ "peer"=>:ip,
277
+ "url"=>:url,
278
+ "range"=>:range,
279
+ "peer_id"=>:string,
280
+ "is_authorized"=>:bool
281
+ }
282
+
283
+ mp["request"]={
284
+ "url"=>:url,
285
+ "range"=>Optional.new(:range)
286
+ }
287
+
288
+ mp["provide"]={
289
+ "url"=>:url,
290
+ "range"=>Optional.new(:range)
291
+ }
292
+
293
+ mp["unrequest"]={
294
+ "url"=>:url,
295
+ "range"=>Optional.new(:range)
296
+ }
297
+
298
+ mp["unprovide"]={
299
+ "url"=>:url,
300
+ "range"=>Optional.new(:range)
301
+ }
302
+
303
+ #the taker sends this message when a transfer finishes
304
+ #if there is an error in the transfer, dont set a hash
305
+ #to signify failure
306
+ #when this is received from the taker, the connection is considered done for all parties
307
+ #
308
+ #The giver also sends this message when they are done transferring.
309
+ #this closes the connection on their side, allowing them to start other transfers
310
+ #It leaves the connection open on the taker side to allow them to decide if the transfer was successful
311
+ #the hash parameter is ignored when sent by the giver
312
+ mp["completed"]={
313
+ #"peer"=>:ip, no longer used
314
+ "url"=>:url,
315
+ "range"=>:range,
316
+ "peer_id"=>:string,
317
+ "hash"=>Optional.new(:string)
318
+ }
319
+
320
+ mp["hash_verify"]={
321
+ "url"=>:url,
322
+ "range"=>:range,
323
+ "hash_ok"=>:bool
324
+ }
325
+
326
+ mp["transfer"]={
327
+ "host"=>:string,
328
+ "port"=>:int,
329
+ "method"=>:string,
330
+ "url"=>:url,
331
+ "range"=>:range,
332
+ "peer_id"=>:string
333
+ }
334
+
335
+ mp["protocol_error"]={
336
+ "message"=>Optional.new(:string)
337
+ }
338
+
339
+ mp["protocol_warn"]={
340
+ "message"=>Optional.new(:string)
341
+ }
342
+
343
+ mp
344
+ end
345
+ end
346
+ end