siefca-htsucker 0.3.0

data/lib/htsucker.rb

@@ -0,0 +1,34 @@
+# encoding: utf-8
+# 
+# HTTP loading and transliteration
+#
+# Author::    Paweł Wilk (mailto:pw@gnu.org)
+# Copyright:: Copyright (c) 2009 Paweł Wilk
+# License::   LGPL
+
+require 'iconv'
+require 'htmlentities'
+require 'net/http'
+require 'net/https'
+require 'timeout'
+require 'uri'
+
+require 'bufferaffects'
+require './htsucker/domains_to_languages'
+require './htsucker/htsucker'
+
+# testing:
+
+sites = []
+sites << 'wykop.pl/wykopalisko'
+sites << 'poland.com'
+sites << 'hyperreal.info'
+sites << 'grono.net'
+sites << 'google.pl'
+sites << 'randomseed.pl'
+sites << 'heise-online.de'
+
+sites.each do |site|
+  pa = HTSucker.new(site)
+  puts "#{pa.real_url}: #{pa.language} #{pa.charset}"
+end

data/lib/htsucker/htsucker.rb

@@ -0,0 +1,468 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+class HTSucker
+
+  include DomainsToLanguages
+  include BufferAffects
+
+  buffers_reset_method  :reset_buffers
+  attr_affects_buffers  :url
+  
+  attr_reader :url
+  
+  # Default options are matrix for defaults used by class method HTSucker.default_options
+  # while setting up class variable @@default_options which is used by instances as a
+  # matrix for options not given when creating new objects.
+  
+  DefaultOpts = { :redir_retry          => 5,
+                  :conn_retry           => 8,
+                  :total_retry          => 2,
+                  :read_timeout         => 15,
+                  :total_timeout        => 30,
+                  :allow_strange_ports  => false,
+                  :max_length           => 524288 }.freeze
+  
+  # Creates new instance of HTSucker. +url+ parameter should be valid URI object or string.
+  # You may want to override defaults by issuing hash containing options you want to be changed.
+  
+  def initialize(url, options=nil)
+    default_options = self.class.default_options.dup
+    if options.respond_to?(:keys)
+      unknown = (options.keys - default_options.keys).join(', ')
+      raise ArgumentError.new("unknown options: #{unknown}") unless unknown.empty?
+      default_options.merge!(options)
+    end
+    default_options.each_pair do |opt_name,opt_value|
+      instance_variable_set("@#{opt_name}", opt_value)
+    end
+    reset_buffers
+    @http_req = nil
+    self.url  = url
+  end
+  
+  # Resets charset and response buffers.
+  
+  def reset_buffers
+    @charset      = nil
+    @content_type = nil
+    @response     = nil
+    @overflow     = 0
+    @real_url     = nil
+  end
+  
+  # Sets new url.
+  
+  def url=(url)
+    url = URI.parse(url) unless url.kind_of? URI
+    url = URI.parse("http://#{url.to_s}") if url.is_a?(URI::Generic) 
+    url.path = '/' if url.path.nil? || url.path.empty?
+    validate_url(url)
+    @url = url
+    @url.freeze
+    @http_req = Net::HTTP::Head.new(@url.path)
+  end
+  
+  # Returns top-level domain for URL.
+  
+  def domain
+    self.url.host.split('.').last.downcase.to_sym
+  end
+  
+  # Returns top-level domain for real URL.
+  
+  def real_domain
+    self.real_url.host.split('.').last.downcase.to_sym
+  end
+  
+  # Returns resource path.
+  def path; url.path end
+  
+  # Returns real resource path.
+  def real_path; real_url.path end
+  
+  # Returns hostname.
+  def host; url.host end
+  
+  # Returns real hostname.
+  def real_host; real_url.host end
+  
+  # Returns used port.
+  def port; url.port end
+
+  # Returns real port.
+  def real_port; real_url.port end
+  
+  # Returns protocol.
+  def protocol; url.class.name.split('::').last.downcase.to_sym end
+  
+  # Returns real protocol.
+  def real_protocol; real_url.class.name.split('::').last.downcase.to_sym end
+
+  # Returns page charset.
+
+  def charset
+    @content_type, @charset = get_page_info if @charset.nil?
+    return @charset
+  end
+  
+  def content_charset;     charset      end
+  def content_charset=(x)  charset=(x)  end
+
+  # Returns page content-type.
+
+  def content_type
+    @content_type, @charset = get_page_info if @content_type.nil?
+    return @content_type
+  end
+  
+  # Returns major name of the content-type or nil if something went wrong.
+  
+  def content_type_major
+    ctype = self.content_type.to_s
+    return nil if ctype.empty?
+    ctype = ctype.split('/').first
+    return nil if ctype.to_s.empty?
+    return ctype.to_sym
+  end
+  
+  # Returns minor name of the content-type or nil if something went wrong.
+  
+  def content_type_minor
+    ctype = self.content_type.to_s
+    return nil if ctype.empty?
+    ctype = ctype.split('/')[1]
+    return nil if ctype.to_s.empty?
+    return ctype.to_sym
+  end
+  
+  def validate_url(url)
+    raise HTSuckerBadURI.new("malformed URI") if url.to_s.empty?
+    u_protocol = url.class.name.split('::').last.upcase
+    unless ['HTTP','HTTPS'].include?(u_protocol)
+      raise HTSuckerBadProtocol.new("bad protocol: #{u_protocol}")
+    end
+    unless @allow_strange_ports
+      if ((u_protocol == 'HTTP' && url.port != 80) ||
+          (u_protocol == 'HTTPS' && url.port != 443))
+        raise HTSuckerBadPort.new("strange port number: #{url.port}")
+      end
+    end
+  end
+  private :validate_url
+  
+  # Translates top-level domain to spoken language code.
+  
+  def domain_to_spoken
+    lang = nil
+    enc = self.content_charset.to_s[0..2].downcase.to_sym
+    national_encodings = [:iso, :win, :"cp-", :koi, :utf]
+    if national_encodings.include?(enc) 
+      lang = @@domain_to_language[self.real_domain] if real_domain.length == 2
+    end
+    return lang
+  end
+  private :domain_to_spoken
+  
+  # Returns content-language or default content language.
+  
+  def content_language(default_content_lanuage='en')
+    clang = nil
+    
+    if self.response.nil?
+      clang = domain_to_spoken
+      return default_content_lanuage
+    end
+    
+    # try meta-tag header
+    unless self.body.to_s.empty? || self.content_type_major != :text
+      header  = body.scan(/<meta http-equiv\s*=\s*['"]*content-language['"]*\s*content\s*=\s*['"]*\s*(.*?)\s*['"]*\s*\/?>/i)
+      header  = header.flatten.first
+      clang   = extract_content_language(header)
+    end
+    
+    # try lang and xml:lang attribute from HTML tag and do the same for body tag
+    if clang.to_s.empty? && !self.body.to_s.empty? && self.content_type_major == :text
+      header  = body.scan(/<x?html\s.*?\s+?lang\s*?=["']*([^"']+).*?\/?>/i)
+      header  = header.flatten.first
+      if header.to_s.empty?
+        header  = body.scan(/<x?html\s.*?\s+?xml:lang\s*?=["']*([^"']+).*?\/?>/i)
+        header  = header.flatten.first
+      end
+      if header.to_s.empty?
+        header  = body.scan(/<body\s.*?\s+?lang\s*?=["']*([^"']+).*?\/?>/i)
+        header  = header.flatten.first
+      end
+      if header.to_s.empty?
+        header  = body.scan(/<body\s.*?\s+?xml:lang\s*?=["']*([^"']+).*?\/?>/i)
+        header  = header.flatten.first
+      end
+      clang = extract_content_language(header)
+    end
+
+    # try server header and in case of 'en' or empty try to figure language by looking at top-domain
+    if clang.to_s.empty? && response.respond_to?(:header)
+      header  = response.header['content-language']
+      clang   = extract_content_language(header)
+      present = clang.to_s
+      clang   = domain_to_spoken if (present.empty? || present[0..1] == 'en')
+      clang   = present if (clang.to_s.empty? && !present.empty?)
+    end
+    
+    # try default
+    clang = default_content_lanuage if clang.to_s.empty?
+    
+    return clang
+  end
+  
+  def language; content_language end
+  def lang;     content_language end
+
+  # Obtains charset from document body or server response header.
+
+  def get_page_info(default_content_type='text/html', default_charset='ascii')
+    return [default_content_type, default_charset] if self.response.nil?
+    
+    # try meta-tag header
+    enc     = nil
+    ctype   = nil
+    
+    # try server header first time to see if we even can analyze the content
+    if response.respond_to?(:header)
+      header = response.header['content-type']
+      first_ctype = extract_content_type(header).to_s.split('/').first
+    end
+    
+    unless (self.body.to_s.empty? || first_ctype != 'text')
+      header  = body.scan(/<meta http-equiv\s*=\s*['"]*content-type['"]*\s*content\s*=\s*['"]*\s*(.*?)\s*['"]*\s*\/?>/i)
+      header  = header.flatten.first
+      enc     = extract_charset(header)
+      ctype   = extract_content_type(header)
+    end
+    
+    # try server header
+    if (ctype.to_s.empty? && response.respond_to?(:header))
+      header  = response.header['content-type']
+      ctype   = extract_content_type(header)
+      enc     = extract_charset(header) if enc.to_s.empty? # weird but may happend (page with charset encoding but without type)
+    end
+    
+    # try defaults
+    enc   = default_charset       if enc.to_s.empty?
+    ctype = default_content_type  if ctype.to_s.empty?
+
+    return [ctype, enc]
+  end
+  private :get_page_info
+
+  # Extracts charset from content-type string.
+
+  def extract_charset(enc_string)
+    return nil if enc_string.nil? || enc_string.empty?
+    ret_enc = nil
+    ct = enc_string.chomp.downcase.squeeze(' ')
+    unless ct.nil?
+      ctary = {}
+      ct.split(';').each do |segment|
+        k,v = segment.split('=')
+        ctary[k.strip.to_sym] = v unless (k.nil? || v.nil?)
+      end
+      if ctary.has_key?(:charset)
+        begin
+          test_enc = ctary[:charset]
+          test_enc = 'utf-8' if test_enc == 'utf8'
+          ret_enc = Encoding.find(test_enc)
+          ret_enc = ret_enc.name
+        rescue ArgumentError
+        end
+      end
+    end
+    ret_enc = nil if ret_enc.nil? || ret_enc.squeeze(" ").empty?
+    return ret_enc.to_s.downcase.to_sym
+  end
+  private :extract_charset
+
+  # Extracts content-type from content-type string.
+
+  def extract_content_type(ctype_string)
+    return nil if ctype_string.to_s.empty?
+    ct = ctype_string.chomp.squeeze(' ').split(';').first
+    ct = ct.strip.downcase.to_sym unless ct.nil?
+    return ct
+  end
+  private :extract_content_type
+
+  # Extracts content-language from content-language string.
+
+  def extract_content_language(ltype_string)
+    return nil if ltype_string.to_s.empty?
+    lt = ltype_string.chomp.squeeze(' ').split(';').first.split(',').first
+    lt = lt.strip.downcase.to_sym unless lt.nil?
+    return lt
+  end
+  private :extract_content_language
+
+
+  # Fetches document using HTTP and returns response object. It also sets charset.
+
+  def response
+    return @response unless @response.nil?
+    url         = @url
+    found       = false
+    response    = nil
+    @real_url   = nil
+    http_req    = @http_req
+    redir_retry = @redir_retry
+    conn_retry  = @conn_retry
+    
+    until found do
+      begin
+        status = Timeout::timeout(@timeout) do
+          case url.scheme.downcase.to_sym
+          when :http
+            response = Net::HTTP.start(url.host, url.port) { |http|  http.request(http_req) }
+          when :https
+            https             = Net::HTTP.new(url.host, url.port)
+            https.use_ssl     = true
+            https.verify_mode = OpenSSL::SSL::VERIFY_NONE
+            response = https.start { |http| http.request(http_req) }
+          else
+            return nil
+          end
+        end
+        response.value
+      rescue Net::HTTPRetriableError
+        conn_retry -= 1
+        if response.respond_to?(:header) && !response.header['location'].nil? && !response.header['location'].empty?
+          url = URI.parse(response.header['location'])
+          validate_url(url)
+          http_req = Net::HTTP::Head.new(url.path)
+          redir_retry -= 1
+        end
+      rescue
+        return nil
+      end
+      if response.kind_of?(Net::HTTPOK)
+        found = true
+        break
+      end
+      break if (redir_retry < 0 || conn_retry < 0)
+    end
+    if found
+      @real_url = url
+      @response = response
+      @content_length = response.header['content-length'].to_s.to_i
+      if @content_length > @max_length
+        raise HTSuckerContentTooBig.new("content length (#{@content_length}) is greater than declared limit (#{@max_length})") 
+      end
+      openuri_opts = { :redirect=>false, :read_timeout=>false }
+      resource = open(@real_url.to_s, openuri_opts)
+      resource.read(@max_length)
+      @content_type, @charset = get_page_info(nil,nil) # using just server headers
+      
+      return response
+    else
+      return nil
+    end
+  end
+  
+  # Returns document body.
+  
+  def body
+    r = self.response
+    return r.respond_to?(:body) ? r.body : nil
+  end
+  
+  # Alias for body.
+  
+  def fetch(*args); body(*args) end
+  
+  # Returns URL used while obtaining content (e.g. after redirection).
+  
+  def real_url
+    return nil if self.response.nil?
+    return @real_url
+  end
+  
+  # Strips HTML tags from document.
+
+  def strip_html(text=nil)
+    text    ||= self.body
+    @coder  ||= HTMLEntities.new
+    r = text.tr("\t", ' ')
+    r.tr!("\r", '')
+    r.sub!(%r{<body.*?>(.*?)</body>}mi, '\1')
+    r.gsub!(%r{<script.*?>(.*?)</script>}mi, ' ')
+    r.gsub!(%r{<style.*?>(.*?)</style>}mi, ' ')
+    r.gsub!(%r{<!--.*?-->}mi, ' ')
+    r.gsub!(/<br\s*\/?>|<p>/mi, "\n")
+    r.gsub!(/<.*?>/m, '')
+    return coder.decode(r)
+  end
+
+  # Transliterates text to ASCII and removes unknown characters.
+
+  def clean_text(text=nil, enc=nil)
+    text            ||= self.body
+    enc             ||= self.charset
+    @transliterator ||= Iconv.new('ASCII//TRANSLIT//IGNORE', 'UTF-8')
+    page = Iconv.iconv('UTF-8//IGNORE', enc, text).join
+    page = strip_html(page)
+    page.gsub!(/['`]/m, '_amp__')
+    page = @transliterator.conv(page).downcase
+    page.tr!(".!?", ' ')
+    page.gsub!(/[^\x00-\x7F]+/, '')
+    page.gsub!(/[^a-z0-9\-_\[\]\(\)\*\=\@\#\$\%\^\&\{\}\:\;\,\<\>\+\s\n\.\!\?]+/im, '')
+    page.gsub!('_amp__',"'")
+    page.squeeze!(" \n")
+    page.gsub!(/^\s?\n\s?$/m, '')
+    page.gsub!(/\n\s/,"\n")
+    page.gsub!(/\s\n/,"\n")
+    page.gsub!(/^\s+/,'')
+    page.gsub!(/(^|\s)\'+(.*?)\'+(\s|$)/m,'\1\2\3')
+    page.gsub!(/(^|\s)\'+(\s|$)/, '')
+    page.squeeze!("\n ")
+    return page
+  end
+  
+  def clean; clean_text end
+
+  # Transliterates text to ASCII and removes unknown characters leaving just words.
+  
+  def clean_words(text=nil, enc=nil)
+    cw = clean_text(text, enc)
+    cw.gsub!(/\[\s*?[^\:]+?\:\/+?.*?\]/mi, ' ')
+    cw.gsub!(/\[\s*?(\d|\s|[^\w])+\]/mi, ' ')
+    cw.gsub!(/[^a-z0-9]+/im, ' ')
+    cw.squeeze!(' ')
+    return cw
+  end
+  
+  # Transliterates text to ASCII, removes unknown characters and returns array of words.
+  
+  def words
+    self.clean_words.split(' ')
+  end
+  
+  # Use this class method to set up default options used when creating new objects.
+  # For each option that you omit it will be taken from constant hash called DefaultOpts.
+  # Default options hash is stored in @@default_options. This method will return current
+  # default options when called without parameter.
+  
+  def self.default_options(opts=nil)
+    @@default_options ||= DefaultOpts.dup
+    return @@default_options.freeze if opts.nil?
+    if opts.respond_to?(:keys)
+      known_opts = DefaultOpts.keys
+      unknown = (opts.keys - known_opts).join(', ')
+      raise ArgumentError.new("unknown options: #{unknown}") unless unknown.empty?
+      @@default_options.unfreeze
+      @@default_options.merge!(opts)
+      return @@default_options.freeze
+    else
+      raise ArgumentError.new("malformed options")
+    end
+  end
+  
+end
+

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification 
+name: siefca-htsucker
+version: !ruby/object:Gem::Version 
+  version: 0.3.0
+platform: ruby
+authors: 
+- "Pawe\xC5\x82 Wilk"
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-04-28 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: bufferaffects
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: HTSucker is simple HTTP(S) reader with ability to transliterate body
+email: pw@gnu.org
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/htsucker.rb
+- lib/htsucker/htsucker.rb
+has_rdoc: true
+homepage: http://randomseed.pl/htsucker
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: HTSucker is simple HTTP(S) reader with ability to transliterate body
+test_files: []
+