lyrics 0.0.2

data/lib/lyrics/utils/formdata.rb

@@ -0,0 +1,56 @@
+# Copyright (C) 2006-2008 by Sergio Pistone
+# sergio_pistone@yahoo.com.ar
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+require "cgi"
+
+module URLEncodedFormData
+
+	def URLEncodedFormData.prepare_query( params )
+		query = params.collect { |name, value| "#{name}=#{CGI.escape( value.to_s() )}" }.join( "&" )
+		header = { "Content-type" => "application/x-www-form-urlencoded" }
+		return query, header
+	end
+
+end
+
+module MultipartFormData
+
+	@@boundary = "----------nOtA5FcjrNZuZ3TMioysxHGGCO69vA5iYysdBTL2osuNwOjcCfU7uiN"
+
+	def MultipartFormData.text_param( name, value )
+		return	"Content-Disposition: form-data; name=\"#{CGI.escape(name)}\"\r\n" \
+				"\r\n" \
+				"#{value}\r\n"
+	end
+
+	def MultipartFormData.file_param( name, file, mime_type, content )
+		return	"Content-Disposition: form-data; name=\"#{CGI.escape(name)}\"; filename=\"#{file}\"\r\n" \
+				"Content-Transfer-Encoding: binary\r\n" \
+				"Content-Type: #{mime_type}\r\n" \
+				"\r\n" \
+				"#{content}\r\n"
+	end
+
+	def MultipartFormData.prepare_query( params )
+		query = params.collect { |param| "--#{@@boundary}\r\n#{param}" }.join( "" ) + "--#{@@boundary}--"
+		header = { "Content-type" => "multipart/form-data; boundary=" + @@boundary }
+		return query, header
+	end
+
+end
+

data/lib/lyrics/utils/htmlentities.rb

@@ -0,0 +1,291 @@
+# Copyright (C) 2006-2008 by Sergio Pistone
+# sergio_pistone@yahoo.com.ar
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+module HTMLEntities
+
+	def HTMLEntities.decode!( text )
+		aux1 = text.gsub!( /&([a-zA-Z]+);/ ) { (n = @@named_entities[$1]) ? [n].pack( "U" ) : $0 }
+		aux2 = text.gsub!( /&#(\d{2,4});/ ) { [$1.to_i()].pack( "U" ) }
+		aux3 = text.gsub!( /&#x([0-9a-fA-F]{2,4});/ ) { [s.to_i(16)].pack( "U" ) }
+		return (aux1||aux2||aux3) ? text : nil
+	end
+
+	def HTMLEntities.decode( text )
+		text = String.new( text )
+		HTMLEntities.decode!( text )
+		return text
+	end
+
+	@@named_entities = {
+		"quot" => 34,
+		"amp" => 38,
+		"lt" => 60,
+		"gt" => 62,
+		"OElig" => 338,
+		"oelig" => 339,
+		"Scaron" => 352,
+		"scaron" => 353,
+		"Yuml" => 376,
+		"circ" => 710,
+		"tilde" => 732,
+		"ensp" => 8194,
+		"emsp" => 8195,
+		"thinsp" => 8201,
+		"zwnj" => 8204,
+		"zwj" => 8205,
+		"lrm" => 8206,
+		"rlm" => 8207,
+		"ndash" => 8211,
+		"mdash" => 8212,
+		"lsquo" => 8216,
+		"rsquo" => 8217,
+		"sbquo" => 8218,
+		"ldquo" => 8220,
+		"rdquo" => 8221,
+		"bdquo" => 8222,
+		"dagger" => 8224,
+		"Dagger" => 8225,
+		"permil" => 8240,
+		"lsaquo" => 8249,
+		"rsaquo" => 8250,
+		"euro" => 8364,
+
+		"fnof" => 402,
+		"Alpha" => 913,
+		"Beta" => 914,
+		"Gamma" => 915,
+		"Delta" => 916,
+		"Epsilon" => 917,
+		"Zeta" => 918,
+		"Eta" => 919,
+		"Theta" => 920,
+		"Iota" => 921,
+		"Kappa" => 922,
+		"Lambda" => 923,
+		"Mu" => 924,
+		"Nu" => 925,
+		"Xi" => 926,
+		"Omicron" => 927,
+		"Pi" => 928,
+		"Rho" => 929,
+		"Sigma" => 931,
+		"Tau" => 932,
+		"Upsilon" => 933,
+		"Phi" => 934,
+		"Chi" => 935,
+		"Psi" => 936,
+		"Omega" => 937,
+		"alpha" => 945,
+		"beta" => 946,
+		"gamma" => 947,
+		"delta" => 948,
+		"epsilon" => 949,
+		"zeta" => 950,
+		"eta" => 951,
+		"theta" => 952,
+		"iota" => 953,
+		"kappa" => 954,
+		"lambda" => 955,
+		"mu" => 956,
+		"nu" => 957,
+		"xi" => 958,
+		"omicron" => 959,
+		"pi" => 960,
+		"rho" => 961,
+		"sigmaf" => 962,
+		"sigma" => 963,
+		"tau" => 964,
+		"upsilon" => 965,
+		"phi" => 966,
+		"chi" => 967,
+		"psi" => 968,
+		"omega" => 969,
+		"thetasym" => 977,
+		"upsih" => 978,
+		"piv" => 982,
+		"bull" => 8226,
+		"hellip" => 8230,
+		"prime" => 8242,
+		"Prime" => 8243,
+		"oline" => 8254,
+		"frasl" => 8260,
+		"weierp" => 8472,
+		"image" => 8465,
+		"real" => 8476,
+		"trade" => 8482,
+		"alefsym" => 8501,
+		"larr" => 8592,
+		"uarr" => 8593,
+		"rarr" => 8594,
+		"darr" => 8595,
+		"harr" => 8596,
+		"crarr" => 8629,
+		"lArr" => 8656,
+		"uArr" => 8657,
+		"rArr" => 8658,
+		"dArr" => 8659,
+		"hArr" => 8660,
+		"forall" => 8704,
+		"part" => 8706,
+		"exist" => 8707,
+		"empty" => 8709,
+		"nabla" => 8711,
+		"isin" => 8712,
+		"notin" => 8713,
+		"ni" => 8715,
+		"prod" => 8719,
+		"sum" => 8721,
+		"minus" => 8722,
+		"lowast" => 8727,
+		"radic" => 8730,
+		"prop" => 8733,
+		"infin" => 8734,
+		"ang" => 8736,
+		"and" => 8743,
+		"or" => 8744,
+		"cap" => 8745,
+		"cup" => 8746,
+		"int" => 8747,
+		"there4" => 8756,
+		"sim" => 8764,
+		"cong" => 8773,
+		"asymp" => 8776,
+		"ne" => 8800,
+		"equiv" => 8801,
+		"le" => 8804,
+		"ge" => 8805,
+		"sub" => 8834,
+		"sup" => 8835,
+		"nsub" => 8836,
+		"sube" => 8838,
+		"supe" => 8839,
+		"oplus" => 8853,
+		"otimes" => 8855,
+		"perp" => 8869,
+		"sdot" => 8901,
+		"lceil" => 8968,
+		"rceil" => 8969,
+		"lfloor" => 8970,
+		"rfloor" => 8971,
+		"lang" => 9001,
+		"rang" => 9002,
+		"loz" => 9674,
+		"spades" => 9824,
+		"clubs" => 9827,
+		"hearts" => 9829,
+		"diams" => 9830,
+
+		"nbsp" => 160,
+		"iexcl" => 161,
+		"cent" => 162,
+		"pound" => 163,
+		"curren" => 164,
+		"yen" => 165,
+		"brvbar" => 166,
+		"sect" => 167,
+		"uml" => 168,
+		"copy" => 169,
+		"ordf" => 170,
+		"laquo" => 171,
+		"not" => 172,
+		"shy" => 173,
+		"reg" => 174,
+		"macr" => 175,
+		"deg" => 176,
+		"plusmn" => 177,
+		"sup2" => 178,
+		"sup3" => 179,
+		"acute" => 180,
+		"micro" => 181,
+		"para" => 182,
+		"middot" => 183,
+		"cedil" => 184,
+		"sup1" => 185,
+		"ordm" => 186,
+		"raquo" => 187,
+		"frac14" => 188,
+		"frac12" => 189,
+		"frac34" => 190,
+		"iquest" => 191,
+		"Agrave" => 192,
+		"Aacute" => 193,
+		"Acirc" => 194,
+		"Atilde" => 195,
+		"Auml" => 196,
+		"Aring" => 197,
+		"AElig" => 198,
+		"Ccedil" => 199,
+		"Egrave" => 200,
+		"Eacute" => 201,
+		"Ecirc" => 202,
+		"Euml" => 203,
+		"Igrave" => 204,
+		"Iacute" => 205,
+		"Icirc" => 206,
+		"Iuml" => 207,
+		"ETH" => 208,
+		"Ntilde" => 209,
+		"Ograve" => 210,
+		"Oacute" => 211,
+		"Ocirc" => 212,
+		"Otilde" => 213,
+		"Ouml" => 214,
+		"times" => 215,
+		"Oslash" => 216,
+		"Ugrave" => 217,
+		"Uacute" => 218,
+		"Ucirc" => 219,
+		"Uuml" => 220,
+		"Yacute" => 221,
+		"THORN" => 222,
+		"szlig" => 223,
+		"agrave" => 224,
+		"aacute" => 225,
+		"acirc" => 226,
+		"atilde" => 227,
+		"auml" => 228,
+		"aring" => 229,
+		"aelig" => 230,
+		"ccedil" => 231,
+		"egrave" => 232,
+		"eacute" => 233,
+		"ecirc" => 234,
+		"euml" => 235,
+		"igrave" => 236,
+		"iacute" => 237,
+		"icirc" => 238,
+		"iuml" => 239,
+		"eth" => 240,
+		"ntilde" => 241,
+		"ograve" => 242,
+		"oacute" => 243,
+		"ocirc" => 244,
+		"otilde" => 245,
+		"ouml" => 246,
+		"divide" => 247,
+		"oslash" => 248,
+		"ugrave" => 249,
+		"uacute" => 250,
+		"ucirc" => 251,
+		"uuml" => 252,
+		"yacute" => 253,
+		"thorn" => 254,
+		"yuml" => 255,
+	}
+
+end

data/lib/lyrics/utils/http.rb

@@ -0,0 +1,198 @@
+# Copyright (C) 2006-2008 by Sergio Pistone
+# sergio_pistone@yahoo.com.ar
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the
+# Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+
+require File.expand_path( File.dirname( __FILE__ ) + "/formdata" )
+
+require "net/http"
+require "uri"
+
+module HTTP
+
+# 	@@user_agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.2) Gecko/20060308 Firefox/1.5.0.2"
+	@@user_agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.9) Gecko/20071110 Firefox/2.0.0.9"
+# 	@@user_agent = "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.8.1.12) Gecko/20080218 Firefox/2.0.0.12"
+
+	@@proxy_url = nil
+	@@proxy_excluded_urls = []
+	@@proxy_reverse = false # if true, excluded_urls list becomes a list with the _only_ urls the proxy should be used for
+
+	def HTTP.normalize_url( url, protocol="http" )
+		url = url.strip()
+		protocol_regexp = /^ *([^: ]+):\/+/
+		md = protocol_regexp.match( url )
+		return nil if md && md[1] != protocol
+		url.gsub!( /\/+$/, "" )				# remove / at the end of the url
+		url.gsub!( protocol_regexp, "" )	# remove the protocol part if there was one
+		return "#{protocol}://#{url}"		# reinsert protocol part assuring protocol:// form
+	end
+
+	def HTTP.set_proxy_settings( proxy_url, excluded_urls=[], reverse=false )
+		@@proxy_url = proxy_url ? HTTP.normalize_url( proxy_url, "http" ) : nil
+		@@proxy_reverse = @@proxy_url ? reverse : false
+		@@proxy_excluded_urls = []
+		if @@proxy_url
+			excluded_urls.each() do |url|
+				url = normalize_url( url, "http" )
+				@@proxy_excluded_urls.insert( -1, url ) if url && ! @@proxy_excluded_urls.include?( url )
+			end
+		end
+	end
+
+	def HTTP.get_proxy_settings()
+		ret = [@@proxy_url ? @@proxy_url.dup : nil, [], @@proxy_reverse]
+		@@proxy_excluded_urls.each() { |url| ret[1][ret[1].size] = url.dup  }
+		return ret
+	end
+
+	# returns proxy_host, proxy_port, proxy_user, proxy_pass for given url
+	def HTTP.get_url_proxy_settings( url )
+
+		return nil, nil, nil, nil if ! @@proxy_url
+		proxy = HTTP.parse_uri( @@proxy_url )
+		return nil, nil, nil, nil if ! proxy.host
+		proxy.port = 80 if ! proxy.port
+
+		# check if given url should be treated specially
+		exception = false
+		@@proxy_excluded_urls.each() do |exception_url|
+			if url.index( exception_url ) == 0
+				exception = true
+				break
+			end
+		end
+
+		if exception && @@proxy_reverse || ! exception && ! @@proxy_reverse
+			return proxy.host, proxy.port, proxy.user, proxy.password
+		else
+			return nil, nil, nil, nil
+		end
+	end
+
+	def HTTP.parse_uri( uri )
+		begin
+			return URI.parse( uri )
+		rescue URI::InvalidURIError
+			return URI.parse( URI.escape( uri ) )
+		end
+	end
+
+	def HTTP.fetch_page_get( url, headers=nil, follow=10 )
+		begin
+
+			p_url = HTTP.parse_uri( url )
+			return nil, url if p_url.host == nil || p_url.port == nil || p_url.request_uri == nil
+			proxy_host, proxy_port, proxy_user, proxy_pass = HTTP.get_url_proxy_settings( url )
+
+			full_headers = {
+				"User-Agent" => @@user_agent,
+				"Referer" => "#{p_url.scheme}://#{p_url.host}",
+			}
+			full_headers.merge!( headers ) if headers
+
+			http = Net::HTTP.new( p_url.host, p_url.port, proxy_host, proxy_port, proxy_user, proxy_pass )
+			response = http.request_get( p_url.request_uri, full_headers )
+
+			case response
+				when Net::HTTPSuccess
+				when Net::HTTPRedirection, Net::HTTPFound
+					if follow == 0
+						response = nil
+					elsif follow > 0
+						response, url = HTTP.fetch_page_get( response["location"], nil, follow-1 )
+					end
+				else
+					response = nil
+			end
+
+			return response, url
+
+		rescue Errno::ETIMEDOUT, Errno::EBADF, EOFError => e
+			raise TimeoutError.new( e.to_s )
+		end
+	end
+
+	def HTTP.fetch_page_post( url, params, headers=nil, follow=10 )
+		begin
+			p_url = HTTP.parse_uri( url )
+			return nil, url if p_url.host == nil || p_url.port == nil || p_url.request_uri == nil
+			proxy_host, proxy_port, proxy_user, proxy_pass = HTTP.get_url_proxy_settings( url )
+
+			data, full_headers = URLEncodedFormData.prepare_query( params )
+			full_headers["User-Agent"] = @@user_agent
+			full_headers["Referer"] = "#{p_url.scheme}://#{p_url.host}"
+			full_headers.merge!( headers ) if headers
+
+			http = Net::HTTP.new( p_url.host, p_url.port, proxy_host, proxy_port, proxy_user, proxy_pass )
+			response = http.request_post( p_url.request_uri, data, full_headers )
+
+			case response
+				when Net::HTTPSuccess
+				when Net::HTTPRedirection, Net::HTTPFound
+					if follow == 0
+						response = nil
+					elsif follow > 0
+						response, url = HTTP.fetch_page_get( response["location"], nil, follow-1 )
+					end
+				else
+					response = nil
+			end
+
+			return response, url
+
+		rescue Errno::ETIMEDOUT, Errno::EBADF, EOFError => e
+			raise TimeoutError.new( e.to_s )
+		end
+	end
+
+	def HTTP.fetch_page_post_form_multipart( url, params, headers=nil, follow=10 )
+		begin
+
+			p_url = HTTP.parse_uri( url )
+			return nil, url if p_url.host == nil || p_url.port == nil || p_url.request_uri == nil
+			proxy_host, proxy_port, proxy_user, proxy_pass = HTTP.get_url_proxy_settings( url )
+
+			data, full_headers = MultipartFormData.prepare_query( params )
+			full_headers["User-Agent"] = @@user_agent
+			full_headers["Referer"] = "#{p_url.scheme}://#{p_url.host}"
+			full_headers.merge!( headers ) if headers
+
+			http = Net::HTTP.new( p_url.host, p_url.port, proxy_host, proxy_port, proxy_user, proxy_pass )
+			response = http.request_post( p_url.request_uri, data, full_headers )
+
+			case response
+				when Net::HTTPSuccess
+				when Net::HTTPRedirection, Net::HTTPFound
+					if follow == 0
+						response = nil
+					elsif follow > 0
+						response, url = HTTP.fetch_page_get( response["location"], nil, follow-1 )
+					end
+				else
+					response = nil
+			end
+
+			return response, url
+
+		rescue Errno::ETIMEDOUT, Errno::EBADF, EOFError => e
+			raise TimeoutError.new( e.to_s )
+		end
+	end
+
+end
+
+