ftw 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,247 @@
1
+ require "cabin" # rubygem "cabin"
2
+ require "net/ftw/dns"
3
+ require "net/ftw/namespace"
4
+ require "socket"
5
+ require "timeout" # ruby stdlib, just for the Timeout exception.
6
+ require "backport-bij" # for Array#rotate, IO::WaitWritable, etc, in ruby < 1.9
7
+
8
+ # A network connection. This is TCP.
9
+ #
10
+ # Example:
11
+ #
12
+ # conn = Net::FTW::Connection.new("www.google.com:80")
13
+ # conn.on(CONNECTED) do |address|
14
+ # puts "Connected to #{address} (#{conn.peer})"
15
+ # conn.write("GET / HTTP/1.0\r\n\r\n")
16
+ # end
17
+ # conn.on(DATA) do |data|
18
+ # puts data
19
+ # end
20
+ # conn.run
21
+ #
22
+ # You can use IO::select on this objects of this type.
23
+ class Net::FTW::Connection2
24
+
25
+ # Events
26
+ CONNECTED = :connected
27
+ DISCONNECTED = :disconnected
28
+ READER_CLOSED = :reader_closed
29
+ DATA = :data
30
+
31
+ # Disconnection reasons
32
+ TIMEOUT = :timeout
33
+ REFUSED = :refused
34
+ LOST = :lost
35
+ INTENTIONAL = :intentional
36
+
37
+ # A new network connection.
38
+ # The 'destination' argument can be an array of strings or a single string.
39
+ # String format is expected to be "host:port"
40
+ #
41
+ # Example:
42
+ #
43
+ # conn = Net::FTW::Connection.new(["1.2.3.4:80", "1.2.3.5:80"])
44
+ #
45
+ # If you specify multiple destinations, they are used in a round-robin
46
+ # decision made during reconnection.
47
+ public
48
+ def initialize(destinations)
49
+ if destinations.is_a?(String)
50
+ @destinations = [destinations]
51
+ else
52
+ @destinations = destinations
53
+ end
54
+
55
+ # Handlers are key => array of callbacks
56
+ @handlers = Hash.new { |h,k| h[k] = [] }
57
+
58
+ @connect_timeout = 2
59
+
60
+ # Use a fixed-size string that we set to BINARY encoding.
61
+ # Not all byte sequences are UTF-8 friendly :0
62
+ @read_size = 16384
63
+ @read_buffer = " " * @read_size
64
+
65
+ # Tell Ruby 1.9 that this string is a binary string, not utf-8 or somesuch.
66
+ if @read_buffer.respond_to?(:force_encoding)
67
+ @read_buffer.force_encoding("BINARY")
68
+ end
69
+
70
+ # TODO(sissel): Validate @destinations
71
+ end # def initialize
72
+
73
+ public
74
+ def connect(timeout=nil)
75
+ # TODO(sissel): Raise if we're already connected?
76
+ close if connected?
77
+ host, port = @destinations.first.split(":")
78
+ @destinations = @destinations.rotate # round-robin
79
+
80
+ # Do dns resolution on the host. If there are multiple
81
+ # addresses resolved, return one at random.
82
+ @remote_address = Net::FTW::DNS.singleton.resolve_random(host)
83
+
84
+ family = @remote_address.include?(":") ? Socket::AF_INET6 : Socket::AF_INET
85
+ @socket = Socket.new(family, Socket::SOCK_STREAM, 0)
86
+ sockaddr = Socket.pack_sockaddr_in(port, @remote_address)
87
+ # TODO(sissel): Support local address binding
88
+
89
+ # Connect with timeout
90
+ begin
91
+ @socket.connect_nonblock(sockaddr)
92
+ rescue IO::WaitWritable
93
+ # Ruby actually raises Errno::EINPROGRESS, but for some reason
94
+ # the documentation says to use this IO::WaitWritable thing...
95
+ # I don't get it, but whatever :(
96
+ if writable?(timeout)
97
+ begin
98
+ @socket.connect_nonblock(sockaddr) # check connection failure
99
+ rescue Errno::EISCONN # Ignore, we're already connected.
100
+ rescue Errno::ECONNREFUSED => e
101
+ # Fire 'disconnected' event with reason :refused
102
+ trigger(DISCONNECTED, :refused, e)
103
+ end
104
+ else
105
+ # Connection timeout
106
+ # Fire 'disconnected' event with reason :timeout
107
+ trigger(DISCONNECTED, :connect_timeout, nil)
108
+ end
109
+ end
110
+
111
+ # We're now connected.
112
+ trigger(CONNECTED, "#{host}:#{port}")
113
+ end # def connect
114
+
115
+ # Is this Connection connected?
116
+ public
117
+ def connected?
118
+ return @connected
119
+ end # def connected?
120
+
121
+ # Write data to this connection.
122
+ # This method blocks until the write succeeds unless a timeout is given.
123
+ #
124
+ # Returns the number of bytes written (See IO#syswrite)
125
+ public
126
+ def write(data, timeout=nil)
127
+ #connect if !connected?
128
+ if writable?(timeout)
129
+ return @socket.syswrite(data)
130
+ else
131
+ raise Timeout::Error.new
132
+ end
133
+ end # def write
134
+
135
+ # Read data from this connection
136
+ # This method blocks until the read succeeds unless a timeout is given.
137
+ #
138
+ # This method is not guaranteed to read exactly 'length' bytes. See
139
+ # IO#sysread
140
+ public
141
+ def read(length, timeout=nil)
142
+ if readable?(timeout)
143
+ begin
144
+ @socket.sysread(length, @read_buffer)
145
+ return @read_buffer
146
+ rescue EOFError
147
+ trigger(READER_CLOSED)
148
+ end
149
+ else
150
+ raise Timeout::Error.new
151
+ end
152
+ end # def read
153
+
154
+ # End this connection
155
+ public
156
+ def disconnect(reason=INTENTIONAL)
157
+ begin
158
+ #@reader_closed = true
159
+ @socket.close_read
160
+ rescue IOError => e
161
+ # Ignore
162
+ end
163
+
164
+ begin
165
+ @socket.close_write
166
+ rescue IOError => e
167
+ # Ignore
168
+ end
169
+
170
+ trigger(DISCONNECTED, reason)
171
+ end # def disconnect
172
+
173
+ # Is this connection writable? Returns true if it is writable within
174
+ # the timeout period. False otherwise.
175
+ #
176
+ # The time out is in seconds. Fractional seconds are OK.
177
+ public
178
+ def writable?(timeout)
179
+ ready = IO.select(nil, [@socket], nil, timeout)
180
+ return !ready.nil?
181
+ end # def writable?
182
+
183
+ # Is this connection readable? Returns true if it is readable within
184
+ # the timeout period. False otherwise.
185
+ #
186
+ # The time out is in seconds. Fractional seconds are OK.
187
+ public
188
+ def readable?(timeout)
189
+ #return false if @reader_closed
190
+ ready = IO.select([@socket], nil, nil, timeout)
191
+ return !ready.nil?
192
+ end # def readable?
193
+
194
+ protected
195
+ def connected(address)
196
+ @remote_address = nil
197
+ @connected = true
198
+ end # def connected
199
+
200
+ protected
201
+ def disconnected(reason, error)
202
+ @remote_address = nil
203
+ @connected = false
204
+ end # def disconnected
205
+
206
+ # The host:port
207
+ public
208
+ def peer
209
+ return @remote_address
210
+ end # def peer
211
+
212
+ # Run this Connection.
213
+ # This is generally meant for Threaded or synchronous operation.
214
+ # For EventMachine, see TODO(sissel): Implement EventMachine support.
215
+ public
216
+ def run
217
+ connect(@connect_timeout) if not connected?
218
+ while connected?
219
+ read_and_trigger
220
+ end
221
+ end # def run
222
+
223
+ # Read data and trigger data callbacks.
224
+ #
225
+ # This is mainly useful if you are implementing your own run loops
226
+ # and IO::select shenanigans.
227
+ public
228
+ def read_and_trigger
229
+ data = read(@read_size)
230
+ if data.length == 0
231
+ disconnect(EOFError)
232
+ else
233
+ trigger(DATA, data)
234
+ end
235
+ end # def read_and_trigger
236
+
237
+ # Support 'to_io' so you can use IO::select on this object.
238
+ public
239
+ def to_io
240
+ return @socket
241
+ end
242
+
243
+ def trigger(*args)
244
+ p :trigger => args
245
+ end
246
+ end # class Net::FTW::Connection
247
+
@@ -0,0 +1,6 @@
1
+ require "net/ftw/namespace"
2
+
3
+ module Net::FTW::CRLF
4
+ # carriage-return + line-feed
5
+ CRLF = "\r\n"
6
+ end
@@ -0,0 +1,57 @@
1
+ require "net/ftw/namespace"
2
+ require "socket"
3
+ # TODO(sissel): Switch to using Resolv::DNS since it lets you (the programmer)
4
+ # choose dns configuration (servers, etc)
5
+ # I still need to wrap whatever Ruby provides because it is historically very
6
+ # inconsistent in implementation behavior across ruby platforms and versions.
7
+ #
8
+ # I didn't really want to write a DNS library.
9
+ class Net::FTW::DNS
10
+ V4_IN_V6_PREFIX = "0:" * 12
11
+
12
+ def self.singleton
13
+ @resolver ||= self.new
14
+ end # def self.singleton
15
+
16
+ # This method is only intended to do A or AAAA lookups
17
+ # I may add PTR lookups later.
18
+ def resolve(hostname)
19
+ official, aliases, family, *addresses = Socket.gethostbyname(hostname)
20
+ # We ignore family, here. Ruby will return v6 *and* v4 addresses in
21
+ # the same gethostbyname() call. It is confusing.
22
+ #
23
+ # Let's just rely entirely on the length of the address string.
24
+ return addresses.collect do |address|
25
+ if address.length == 16
26
+ unpack_v6(address)
27
+ else
28
+ unpack_v4(address)
29
+ end
30
+ end
31
+ end # def resolve
32
+
33
+ def resolve_random(hostname)
34
+ addresses = resolve(hostname)
35
+ return addresses[rand(addresses.size)]
36
+ end # def resolve_random
37
+
38
+ private
39
+ def unpack_v4(address)
40
+ return address.unpack("C4").join(".")
41
+ end # def unpack_v4
42
+
43
+ private
44
+ def unpack_v6(address)
45
+ if address.length == 16
46
+ # Unpack 16 bit chunks, convert to hex, join with ":"
47
+ address.unpack("n8").collect { |p| p.to_s(16) } \
48
+ .join(":").sub(/(?:0:(?:0:)+)/, "::")
49
+ else
50
+ # assume ipv4
51
+ # Per the following sites, "::127.0.0.1" is valid and correct
52
+ # http://en.wikipedia.org/wiki/IPv6#IPv4-mapped_IPv6_addresses
53
+ # http://www.tcpipguide.com/free/t_IPv6IPv4AddressEmbedding.htm
54
+ "::" + unpack_v4(address)
55
+ end
56
+ end # def unpack_v6
57
+ end # class Net::FTW::DNS
@@ -0,0 +1,2 @@
1
+ require "net/ftw/namespace"
2
+ require "net/ftw/http/client"
@@ -0,0 +1,116 @@
1
+ require "net/ftw/http/connection"
2
+ require "net/ftw/http/request"
3
+ require "net/ftw/http/response"
4
+ require "net/ftw/namespace"
5
+ require "socket" # ruby stdlib
6
+
7
+ # TODO(sissel): Split this out into a general 'client' class (outside http)
8
+ # TODO(sissel): EventMachine support
9
+
10
+ # A client should be like a web browser. It should support lots of active
11
+ # connections.
12
+ class Net::FTW::HTTP::Client
13
+ include Net::FTW::CRLF
14
+
15
+ # Create a new HTTP client. You probably only need one of these.
16
+ def initialize
17
+ @connections = []
18
+ end # def initialize
19
+
20
+ # TODO(sissel): This method may not stay. I dunno yet.
21
+ public
22
+ def get(uri, headers={})
23
+ # TODO(sissel): enforce uri scheme options? (ws, wss, http, https?)
24
+ prepare("GET", uri, headers)
25
+ end # def get
26
+
27
+ public
28
+ def prepare(method, uri, headers={})
29
+ uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
30
+ uri.port ||= 80
31
+
32
+ request = Net::FTW::HTTP::Request.new(uri)
33
+ response = Net::FTW::HTTP::Response.new
34
+ request.method = method
35
+ request.version = 1.1
36
+ headers.each do |key, value|
37
+ request.headers[key] = value
38
+ end
39
+
40
+ # TODO(sissel): This is starting to feel like not the best way to implement
41
+ # protocols.
42
+ connection = Net::FTW::HTTP::Connection.new("#{uri.host}:#{uri.port}")
43
+ connection.on(connection.class::CONNECTED) do |address|
44
+ connection.write(request.to_s)
45
+ connection.write(CRLF)
46
+ end
47
+ connection.on(connection.class::HEADERS_COMPLETE) do |version, status, headers|
48
+ response.status = status
49
+ response.version = version
50
+ headers.each { |field, value| response.headers.add(field, value) }
51
+
52
+ # TODO(sissel): Split these BODY handlers into separate body-handling
53
+ # classes.
54
+ if response.headers.include?("Content-Length")
55
+ length = response.headers.get("Content-Length").to_i
56
+ connection.on(connection.class::MESSAGE_BODY) do |data|
57
+ length -= data.size
58
+ #$stdout.write data
59
+ if length <= 0
60
+ if response.headers.get("Connection") == "close"
61
+ connection.disconnect
62
+ else
63
+ p :response_complete => response.headers.get("Content-Length")
64
+ # TODO(sissel): This connection is now ready for another HTTP
65
+ # request.
66
+ end
67
+
68
+ # TODO(sissel): What to do with the extra bytes?
69
+ if length < 0
70
+ # Length is negative, will be offset on end of data string
71
+ $stderr.puts :TOOMANYBYTES => data[length .. -1]
72
+ end
73
+ end
74
+ end
75
+ elsif response.headers.get("Transfer-Encoding") == "chunked"
76
+ connection.on(connection.class::MESSAGE_BODY) do |data|
77
+ # TODO(sissel): Handle chunked encoding
78
+ p :chunked => data
79
+ end
80
+ elsif response.version == 1.1
81
+ # No content-length nor transfer-encoding. If this is HTTP/1.1, this is
82
+ # an error, I think. I need to find the specific part of RFC2616 that
83
+ # specifies this.
84
+ connection.disconnect("Invalid HTTP Response received. Response " \
85
+ "version claimed 1.1 but no Content-Length nor Transfer-Encoding "\
86
+ "header was set in the response.")
87
+ end
88
+ end # connection.on HEADERS_COMPLETE
89
+ #connection.run
90
+ return connection
91
+ end # def prepare
92
+
93
+ def prepare2(method, uri, headers={})
94
+ uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
95
+ uri.port ||= 80
96
+
97
+ request = Net::FTW::HTTP::Request.new(uri)
98
+ response = Net::FTW::HTTP::Response.new
99
+ request.method = method
100
+ request.version = 1.1
101
+ headers.each do |key, value|
102
+ request.headers[key] = value
103
+ end
104
+
105
+ # TODO(sissel): This is starting to feel like not the best way to implement
106
+ # protocols.
107
+ id = "#{uri.scheme}://#{uri.host}:#{uri.port}/..."
108
+ connection = Net::FTW::HTTP::Connection.new("#{uri.host}:#{uri.port}")
109
+ @connections[id] = connection
110
+ end # def prepare2
111
+
112
+ # TODO(sissel):
113
+ def run
114
+ # Select across all active connections, do read_and_trigger, etc.
115
+ end # def run
116
+ end # class Net::FTW::HTTP::Client
@@ -0,0 +1,80 @@
1
+ require "net/ftw/connection2"
2
+ require "net/ftw/http/request"
3
+ require "net/ftw/http/response"
4
+ require "net/ftw/namespace"
5
+ require "socket" # ruby stdlib
6
+
7
+ # TODO(sissel): Split this out into a general 'client' class (outside http)
8
+ # TODO(sissel): EventMachine support
9
+
10
+ # A client should be like a web browser. It should support lots of active
11
+ # connections.
12
+ class Net::FTW::HTTP::Client2
13
+ include Net::FTW::CRLF
14
+
15
+ # Create a new HTTP client. You probably only need one of these.
16
+ def initialize
17
+ @connections = []
18
+ end # def initialize
19
+
20
+ # TODO(sissel): This method may not stay. I dunno yet.
21
+ public
22
+ def get(uri, headers={})
23
+ # TODO(sissel): enforce uri scheme options? (ws, wss, http, https?)
24
+ return prepare("GET", uri, headers)
25
+ end # def get
26
+
27
+ public
28
+ def prepare(method, uri, headers={})
29
+ uri = Addressable::URI.parse(uri.to_s) if uri.is_a?(URI)
30
+ uri.port ||= 80
31
+
32
+ request = Net::FTW::HTTP::Request.new(uri)
33
+ response = Net::FTW::HTTP::Response.new
34
+ request.method = method
35
+ request.version = 1.1
36
+ headers.each do |key, value|
37
+ request.headers[key] = value
38
+ end
39
+
40
+ connection = Net::FTW::Connection2.new("#{uri.host}:#{uri.port}")
41
+ return fiberup(connection, request, response)
42
+ end # def prepare
43
+
44
+ def fiberup(connection, request, response)
45
+ # Body just passes through
46
+ body = Fiber.new do |data|
47
+ Fiber.yield data
48
+ end
49
+
50
+ # Parse the HTTP headers
51
+ headers = Fiber.new do |data|
52
+ parser = HTTP::Parser.new
53
+ headers_done = false
54
+ parser.on_headers_complete = proc { headers_done = true; :stop }
55
+ while true do
56
+ offset = parser << data
57
+ if headers_done
58
+ version = "#{parser.http_major}.#{parser.http_minor}".to_f
59
+ p :processing
60
+ Fiber.yield [version, parser.status_code, parser.headers]
61
+ p :processing
62
+ # Transfer control to the 'body' fiber.
63
+ body.transfer(data[offset..-1])
64
+ end
65
+ p :waiting
66
+ data = Fiber.resume
67
+ end
68
+ end
69
+
70
+ connect = Fiber.new do
71
+ connection.connect
72
+ connection.write(request.to_s + CRLF)
73
+ while true do
74
+ data = connection.read(16384)
75
+ headers.resume data
76
+ end
77
+ end
78
+ return connect
79
+ end # def fiberup
80
+ end # class Net::FTW::HTTP::Client2