RubyGems - webpage - Versions diffs - 0.0.4 → 0.0.5 - Mend

webpage 0.0.4 → 0.0.5

Files changed (2) hide show

data/webpage.rb +121 -276
metadata +6 -4

data/webpage.rb CHANGED Viewed

@@ -2,315 +2,160 @@
 require 'pp'
 require 'mechanize'
 require 'uri'
-class Webpage
-    attr_reader :links,:relative_paths,:outbound_links,:successful,:related_uris,:invalid_links,:internal_links,:internal_outbound_links,:internal_inbound_links,:broken_outbound_links,:external_outbound_links,:external_inbound_links
-    attr_accessor :ignored_exts
-    def initialize(uri)
-        @links = Array.new
-        @relative_paths = Array.new
-        @outbound_links = Array.new
-        @internal_outbound_links = Array.new
-        @external_outbound_links = Array.new
-        @broken_outbound_links = Array.new
-        @external_inbound_links = Array.new
-        @back_links = Array.new
-        @internal_inbound_links = Array.new
-        @external_inbound_links = Array.new
-        @internal_links = Array.new
-        @invalid_links = Array.new
-        @accessed_uri = Array.new
-        @related_uris = Array.new
-        @successful = false
-        begin
-            @uri = URI.parse(uri)
-            raise 'not url' unless @uri.class == URI::HTTP or @uri.class == URI::HTTPS
-            @domain = Webpage.host_to_domain @uri.host
-            agent = Mechanize.new
-            agent.open_timeout = 3
-            @page = agent.get @uri.to_s
-            raise 'not webpage' unless @page.class == Mechanize::Page
-            @page.body = @page.body.force_encoding(@page.encoding).encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "?")
-            @successful = true
-        rescue Exception => e
-            warn "................\nget #{@uri} failed\n.#{e.backtrace.join("\n")}\n #{e}\nURI:.............."
-        end
-    end
-    def encoding
-        return @page.encoding
-    end
-    def keywords
-        meta = @page.search("//meta[@name='keywords']").first
-        return meta.attributes["content"].value.split(',') unless meta.nil?
-    end
-    def description
-        meta = @page.search("//meta[@name='description']").first
-        if meta.nil?
-            return false
-        end
-        return meta.atrributes['content'].value
-    end
-    def body
-        return @page.body
-        #(return @page.body unless @page.body.include?'<html>') if @successful
-        #return String.new
-    end
-    def text
-        return Nokogiri::HTML(body).xpath("//text()").text
-        #return body.gsub(/<\/?[^>]*>/, "")
-    end
-    def title
-        return @page.title unless @page.title.nil?
-        return false
-    end
-    #get all links from html content
-    #1.$all = get all <a>
-    #2.$href = get all href from $all
-    #3.make all $href to be absolute path and put to @links
-=begin
-    def links
-        return @links unless @links.empty?
-        begin
-            agent = Mechanize.new
-            agent.open_timeout = 5
-            agent.get @uri do |page|
-                page.links.each do |link| #1
-                    next if link.href.nil?
-                    uri = Webpage.uri_normalize(link.href)
-                    begin
-                        @links << @uri.merge(uri).to_s
-                    rescue URI::InvalidURIError,URI::InvalidComponentError
-                        warn "ignore\n #{uri} \n #{link.href}"
-                    end
-                end
-            end
-        rescue Errno::ETIMEDOUT,Timeout::Error
-            warn "timeout:#{@uri}"
-        rescue NoMethodError => e
-            warn "no method, mechanize recognize this as a file:#{@uri}.#{e}"
-        rescue Zlib::GzipFile::Error,Mechanize::Error => e
-            warn "gzip error:#{@uri}.#{e}"
-        rescue Net::HTTP::Persistent::Error
-            warn "network reset:#{@uri}"
-        rescue SocketError =>e
-            warn "#{e}.#{@uri}"
-        end
-        return Array.new if @links.empty?
-        #@links = @links.uniq - @accessed_uri
-        #@accessed_uri += @links
-        @links.uniq!
-        scan_links
-        return @links
-    end
-=end
-    def report
-        scan_links
-        scan_outbound_links
-        scan_inbound_links
-        report = {
-            :internal_links => @internal_links,
-            :internal_outbound_links => @internal_outbound_links,
-            :outbound_links => @outbound_links,
-            :broken_outbound_links => @broken_outbound_links,
-            :external_inbound_links => @external_inbound_links,
-            :internal_inbound_links => @internal_inbound_links,
-            :external_outbound_links => @external_outbound_links,
-            :related_uris => @related_uris,
-            :invalid_links => @invalid_links
-        }
-    end
-=begin
-    def external_outbound_links
-        return @external_outbound_links  unless @external_outbound_links.empty?
-        links
-        return @external_outbound_links
-    end
-    def internal_outbound_links
-        return @internal_outbound_links unless @internal_outbound_links.empty?
-        links
-        return @internal_outbound_links
-    end
-    def back_links#inbound links among all the outbound links
-        return @back_links unless @back_links.empty?
-        scan_outbound_links
-        return @back_links
-    end
-    def broken_outbound_links
-        return @broken_outbound_links unless @broken_outbound_links.empty?
-        scan_outbound_links
-        return @broken_outbound_links
-    end
-    def external_inbound_links#outter inbound links
-        return @external_inbound_links unless @external_inbound_links.empty?
-        scan_inbound_links
-        return @external_inbound_links
-    end
-    def internal_inbound_links
-        return @internal_inbound_links unless @internal_inbound_links.empty?
-        scan_inbound_links
-        return @internal_inbound_links
-    end
-=end
-    def pagerank
-        return @pagerank unless @pagerank.nil?
-        require 'page_rankr'
-        @pagerank = PageRankr.ranks(@uri.to_s, :google)[:google]
-        return @pagerank
-    end
-    def ppl#pagerank per link
-        pagerank
-        return false if @pagerank.nil?
-        scan_links
-        return (@pagerank / @links.size)
-    end
-    def scan_related_uris(related_keywords=Array.new,seed_uris=Array.new,checked_uris=Array.new,max=100)#todo: multi-threads
-        scan_links
-        raise "related_keywords is not array,but a #{related_keywords.class}" unless related_keywords.class == Array and seed_uris.class == Array and checked_uris.class == Array
-        seed_uris.concat(@external_outbound_links - checked_uris)
-        related_keywords.concat(keywords)
-        result = Array.new
-        while seed_uris.size > 0 and result.size < max
-            uri = seed_uris.first
-            checked_uris << uri unless checked_uris.include?uri
-            seed_uris.delete(uri)
-            w = Webpage.new uri
-            next unless w.successful
-            text = w.body + w.title
-            related_keywords.each do |word|
-                if text.include?word
-                    #result.concat self.the_related_uris(related_keywords,seed_uris,checked_uris,max)
-                    domain = Webpage.host_to_domain(URI.parse(uri).host)
-                    result << domain unless result.include? domain
-                    seed_uris.concat(w.external_outbound_links - checked_uris)
-                    break
-                end
-            end
-        end
-        return result
-    end
-    def link_to(target_uri)
-        scan_links
-        target_uri = Webpage.uri_normalize(target_uri)
-        target_host = URI.parse(target_uri).host
-        target_domain = Webpage.host_to_domain(target_host)
-        type = 0 #not link to
-        @links.each do |link|
-            candidate_host = URI.parse(link).host
-            if link == target_uri
-                type = 3 #definitely link to
-                break
-            elsif  URI.parse(link).host == target_host
-               type = 2 if type < 2 #link to the host
-            elsif Webpage.host_to_domain(candidate_host) == target_domain
-               type = 1 if type < 1 #link to the root domain
-            end
-        end
-        return type
-    end
+class WebHelper
     def self.uri_normalize(uri)
         uri = URI.parse(uri).normalize
         fragment = uri.fragment
-        return uri.to_s.delete("##{fragment}")
+        uri = uri.to_s
+        uri.sub!(/##{fragment}$/,'') unless fragment.nil?
+        return uri
         #uri = uri.to_s.strip.sub(/\#.*$/,'')
         #uri.path = '/' if uri.path.nil?
     end
     def self.host_to_domain(host)
         domain = (host.match /\.?([a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+)$/)
         return domain[1] unless domain.nil?
         return false
     end
     def self.uri_encode(str)
         return URI.encode(str,Regexp.new("[^#{URI::PATTERN::UNRESERVED+'#:/?%&='}]"))
     end
+end
-    def scan_inbound_links
-        scan_links
-        @back_links.each do |inlink|
-            inlink = URI.parse inlink
-            if @domain == Webpage.host_to_domain(inlink.host)
-                @internal_inbound_links << inlink.to_s
-            else
-                @external_inbound_links << inlink.to_s
-            end
+class Mechanize::Page
+    #@invalid_links = Hash.new
+    attr_reader :valid_links,:invalid_links,:outbound_links,:internal_outbound_links,:external_outbound_links
+    public
+    def text
+        return Nokogiri::HTML(body).xpath("//text()").text
+        #return body.gsub(/<\/?[^>]*>/, "")
+    end
+    def keywords
+        meta = search("//meta[@name='keywords']").first
+        return meta.attributes["content"].value.split(',') unless meta.nil?
+    end
+    def description
+        meta = search("//meta[@name='description']").first
+        if meta.nil?
+            return false
         end
-        @internal_inbound_links.uniq!
-        @external_inbound_links.uniq!
+        return meta.attributes['content'].value
     end
-    def scan_outbound_links
-        @outbound_links.each do |outlink|
-            w = Webpage.new(outlink)
-            unless w.successful
-                @invalid_links << outlink
-                next
-            end
-            next if w.links.nil?
-            w.links.each do |uri|
-                #uri = URI.parse(uri)
-                #next if uri.host.nil?
-                if Webpage.host_to_domain(uri) == @domain
-                    @back_links << uri.to_s
-                else
-                    @broken_outbound_links << uri.to_s
-                end
-            end
-        end
-        @back_links.uniq!
-        @broken_outbound_links.uniq!
+    def pagerank
+        require 'page_rankr'
+        @pagerank = PageRankr.ranks(@uri.to_s, :google)[:google]
+        return @pagerank
     end
     def scan_links
-        return unless @links.empty?
+        @external_outbound_links = Array.new
+        @internal_outbound_links = Array.new
+        @valid_links = Array.new
+        @invalid_links = Array.new
+        @nofollowed_links = Array.new
         exts_to_ignored = %w(.exe .jpg .png .gif .msi .pdf .swf) #decide before download the uri
-        @page.links.each do |link|
+        links.each do |link|
             #初步解析
-            begin
-                uri = URI.parse(link.href)
-                href = uri.to_s
-            rescue URI::InvalidURIError => e
-                @invalid_links << href
+=begin
+    uri = URI.parse(link.uri).normalize
+    href = uri.to_s
+rescue URI::InvalidURIError => e
+    pp link
+    puts e
+    @invalid_links << link
+    next
+=end
+            #忽略非http请求
+            if link.uri.respond_to?'scheme' and !link.uri.scheme.nil? and link.uri.scheme != 'http' and link.uri.scheme != 'https'
+                @invalid_links << link#todo 不同链接key重复，无法体现
+                next
+            end
+            #忽略非网页文件,忽略js按钮忽略邮件
+            if !link.href.nil? and exts_to_ignored.include?link.href[-4,4]# or href.start_with?'javascript:' or href.start_with?'mailto:'
+                @invalid_links << link
                 next
             end
-            #忽略非网页文件
-            if exts_to_ignored.include?href[-4,4]
-                @ignored_uris << href
+            #nofollow links
+            if link.rel.include?'nofollow'
+                @nofollowed_links << link
                 next
             end
+            if link.respond_to?'fragment' and link.fragment.empty?
+                @invalid_links << link
+                next
+            end
+            pp link
             #处理相对路径
-            if uri.relative?
-                @relative_paths << href
-                href = @uri.merge(href).to_s
-                @internal_outbound_links << href
+            if !link.uri.nil? and link.uri.relative?
+                @invalid_links << link
+                #puts @uri.merge(link)
+                #link.uri = @uri.merge(link.uri)
+                @internal_outbound_links << link unless link.uri == @uri
+            elsif link.uri.nil?
+                warn "warning: host nil #{link.uri}"
+                next
             else
-                href = Webpage.uri_normalize(href)
-                if Webpage.host_to_domain(uri.host) == @domain
-                    @internal_outbound_links << href
+                if link.uri.to_s.start_with?'/' or @uri.merge(link.uri).domain == @uri.domain
+                    @internal_outbound_links << link
                 else
-                    @external_outbound_links << href
+                    @external_outbound_links << link
                 end
             end
-            @links << href
+            @valid_links << link
+        end
+        @outbound_links = @internal_outbound_links + @external_outbound_links
+        @scanned = true
+    end
+end
+class URI::Generic
+    def absolute?()
+        if @scheme or path.start_with?'/'
+            true
+        else
+            false
+        end
+    end
+    def domain
+        domain = (host.match /\.?([a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+)$/)
+        return domain[1] unless domain.nil?
+        return nil
+    end
+=begin
+    def normalize!
+        if path && path == ''
+            set_path('/')
+        end
+        if scheme && scheme != scheme.downcase
+            set_scheme(self.scheme.downcase)
         end
-        @outbound_links = @internal_outbound_links.uniq! + @external_outbound_links.uniq!
+        if host && host != host.downcase
+            set_host(self.host.downcase)
+        end
+        set_fragment(nil) unless fragment.nil?
     end
+=end
 end
+=begin
+class URI::Parser
+    def parse(uri)
+        scheme, userinfo, host, port, registry, path, opaque, query, fragment = self.split(uri)
+        if scheme && URI.scheme_list.include?(scheme.upcase)
+            URI.scheme_list[scheme.upcase].new(scheme, userinfo, host, port, registry, path, opaque, query, nil, self)
+        else
+            URI::Generic.new(scheme, userinfo, host, port, registry, path, opaque, query, nil, self)
+        end
+    end
+end
+a = Mechanize.new
+w = a.get('http://dict.youdao.com/w/abc/')
+w.scan_links
+pp w.internal_outbound_links
+exit
+w.links.each do |link|
+    puts link.rel
+end
+=end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webpage
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease:
 platform: ruby
 authors:
@@ -11,14 +11,15 @@ bindir: bin
 cert_chain: []
 date: 2012-04-11 00:00:00.000000000 Z
 dependencies: []
-description: to show seo oriented reports of the webpage,newbie's work, careful
+description: modify Mechanize::Page to show seo oriented reports of the webpage,newbie's
+  work, careful
 email: seoaqua@qq.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - webpage.rb
-homepage: http://seoaqua.com
+homepage: http://github.com/seoaqua/ruby-webpage
 licenses: []
 post_install_message:
 rdoc_options: []
@@ -41,5 +42,6 @@ rubyforge_project:
 rubygems_version: 1.8.21
 signing_key:
 specification_version: 3
-summary: to show seo oriented reports of the webpage,newbie's work, careful
+summary: modify Mechanize::Page to show seo oriented reports of the webpage,newbie's
+  work, careful
 test_files: []