RubyGems - webpage - Versions diffs - 0.0.4 → 0.0.5 - Mend

webpage 0.0.4 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (2) hide show

data/webpage.rb +121 -276
metadata +6 -4

data/webpage.rb CHANGED Viewed

@@ -2,315 +2,160 @@
 require 'pp'
 require 'mechanize'
 require 'uri'
-class Webpage
-    attr_reader :links,:relative_paths,:outbound_links,:successful,:related_uris,:invalid_links,:internal_links,:internal_outbound_links,:internal_inbound_links,:broken_outbound_links,:external_outbound_links,:external_inbound_links
-    attr_accessor :ignored_exts
-    def initialize(uri)
-        @links = Array.new
-        @relative_paths = Array.new
-        @outbound_links = Array.new
-        @internal_outbound_links = Array.new
-        @external_outbound_links = Array.new
-        @broken_outbound_links = Array.new
-        @external_inbound_links = Array.new
-        @back_links = Array.new
-        @internal_inbound_links = Array.new
-        @external_inbound_links = Array.new
-        @internal_links = Array.new
-        @invalid_links = Array.new
-        @accessed_uri = Array.new
-        @related_uris = Array.new
-        @successful = false
-        begin
-            @uri = URI.parse(uri)
-            raise 'not url' unless @uri.class == URI::HTTP or @uri.class == URI::HTTPS
-            @domain = Webpage.host_to_domain @uri.host
-            agent = Mechanize.new
-            agent.open_timeout = 3
-            @page = agent.get @uri.to_s
-            raise 'not webpage' unless @page.class == Mechanize::Page
-            @page.body = @page.body.force_encoding(@page.encoding).encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "?")
-            @successful = true
-        rescue Exception => e
-            warn "................\nget #{@uri} failed\n.#{e.backtrace.join("\n")}\n #{e}\nURI:.............."
-        end
-    end
-    def encoding
-        return @page.encoding
-    end
-    def keywords
-        meta = @page.search("//meta[@name='keywords']").first
-        return meta.attributes["content"].value.split(',') unless meta.nil?
-    end
-    def description
-        meta = @page.search("//meta[@name='description']").first
-        if meta.nil?
-            return false
-        end
-        return meta.atrributes['content'].value
-    end
-    def body
-        return @page.body
-        #(return @page.body unless @page.body.include?'<html>') if @successful
-        #return String.new
-    end
-    def text
-        return Nokogiri::HTML(body).xpath("//text()").text
-        #return body.gsub(/<\/?[^>]*>/, "")
-    end
-    def title
-        return @page.title unless @page.title.nil?
-        return false
-    end
-    #get all links from html content
-    #1.$all = get all <a>
-    #2.$href = get all href from $all
-    #3.make all $href to be absolute path and put to @links
-=begin
-    def links
-        return @links unless @links.empty?
-        begin
-            agent = Mechanize.new
-            agent.open_timeout = 5
-            agent.get @uri do |page|
-                page.links.each do |link| #1
-                    next if link.href.nil?
-                    uri = Webpage.uri_normalize(link.href)
-                    begin
-                        @links << @uri.merge(uri).to_s
-                    rescue URI::InvalidURIError,URI::InvalidComponentError
-                        warn "ignore\n #{uri} \n #{link.href}"
-                    end
-                end
-            end
-        rescue Errno::ETIMEDOUT,Timeout::Error
-            warn "timeout:#{@uri}"
-        rescue NoMethodError => e
-            warn "no method, mechanize recognize this as a file:#{@uri}.#{e}"
-        rescue Zlib::GzipFile::Error,Mechanize::Error => e
-            warn "gzip error:#{@uri}.#{e}"
-        rescue Net::HTTP::Persistent::Error
-            warn "network reset:#{@uri}"
-        rescue SocketError =>e
-            warn "#{e}.#{@uri}"
-        end
-        return Array.new if @links.empty?
-        #@links = @links.uniq - @accessed_uri
-        #@accessed_uri += @links
-        @links.uniq!
-        scan_links
-        return @links
-    end
-=end
-    def report
-        scan_links
-        scan_outbound_links
-        scan_inbound_links
-        report = {
-            :internal_links => @internal_links,
-            :internal_outbound_links => @internal_outbound_links,
-            :outbound_links => @outbound_links,
-            :broken_outbound_links => @broken_outbound_links,
-            :external_inbound_links => @external_inbound_links,
-            :internal_inbound_links => @internal_inbound_links,
-            :external_outbound_links => @external_outbound_links,
-            :related_uris => @related_uris,
-            :invalid_links => @invalid_links
-        }
-    end
-=begin
-    def external_outbound_links
-        return @external_outbound_links  unless @external_outbound_links.empty?
-        links
-        return @external_outbound_links
-    end
-    def internal_outbound_links
-        return @internal_outbound_links unless @internal_outbound_links.empty?
-        links
-        return @internal_outbound_links
-    end
-    def back_links#inbound links among all the outbound links
-        return @back_links unless @back_links.empty?
-        scan_outbound_links
-        return @back_links
-    end
-    def broken_outbound_links
-        return @broken_outbound_links unless @broken_outbound_links.empty?
-        scan_outbound_links
-        return @broken_outbound_links
-    end
-    def external_inbound_links#outter inbound links
-        return @external_inbound_links unless @external_inbound_links.empty?
-        scan_inbound_links
-        return @external_inbound_links
-    end
-    def internal_inbound_links
-        return @internal_inbound_links unless @internal_inbound_links.empty?
-        scan_inbound_links
-        return @internal_inbound_links
-    end
-=end
-    def pagerank
-        return @pagerank unless @pagerank.nil?
-        require 'page_rankr'
-        @pagerank = PageRankr.ranks(@uri.to_s, :google)[:google]
-        return @pagerank
-    end
-    def ppl#pagerank per link
-        pagerank
-        return false if @pagerank.nil?
-        scan_links
-        return (@pagerank / @links.size)
-    end
-    def scan_related_uris(related_keywords=Array.new,seed_uris=Array.new,checked_uris=Array.new,max=100)#todo: multi-threads
-        scan_links
-        raise "related_keywords is not array,but a #{related_keywords.class}" unless related_keywords.class == Array and seed_uris.class == Array and checked_uris.class == Array
-        seed_uris.concat(@external_outbound_links - checked_uris)
-        related_keywords.concat(keywords)
-        result = Array.new
-        while seed_uris.size > 0 and result.size < max
-            uri = seed_uris.first
-            checked_uris << uri unless checked_uris.include?uri
-            seed_uris.delete(uri)
-            w = Webpage.new uri
-            next unless w.successful
-            text = w.body + w.title
-            related_keywords.each do |word|
-                if text.include?word
-                    #result.concat self.the_related_uris(related_keywords,seed_uris,checked_uris,max)
-                    domain = Webpage.host_to_domain(URI.parse(uri).host)
-                    result << domain unless result.include? domain
-                    seed_uris.concat(w.external_outbound_links - checked_uris)
-                    break
-                end
-            end
-        end
-        return result
-    end
-    def link_to(target_uri)
-        scan_links
-        target_uri = Webpage.uri_normalize(target_uri)
-        target_host = URI.parse(target_uri).host
-        target_domain = Webpage.host_to_domain(target_host)
-        type = 0 #not link to
-        @links.each do |link|
-            candidate_host = URI.parse(link).host
-            if link == target_uri
-                type = 3 #definitely link to
-                break
-            elsif  URI.parse(link).host == target_host
-               type = 2 if type < 2 #link to the host
-            elsif Webpage.host_to_domain(candidate_host) == target_domain
-               type = 1 if type < 1 #link to the root domain
-            end
-        end
-        return type
-    end
+class WebHelper
     def self.uri_normalize(uri)
         uri = URI.parse(uri).normalize
         fragment = uri.fragment
-        return uri.to_s.delete("##{fragment}")
+        uri = uri.to_s
+        uri.sub!(/##{fragment}$/,'') unless fragment.nil?
+        return uri
         #uri = uri.to_s.strip.sub(/\#.*$/,'')
         #uri.path = '/' if uri.path.nil?
     end
     def self.host_to_domain(host)
         domain = (host.match /\.?([a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+)$/)
         return domain[1] unless domain.nil?
         return false
     end
     def self.uri_encode(str)
         return URI.encode(str,Regexp.new("[^#{URI::PATTERN::UNRESERVED+'#:/?%&='}]"))
     end
+end
-    def scan_inbound_links
-        scan_links
-        @back_links.each do |inlink|
-            inlink = URI.parse inlink
-            if @domain == Webpage.host_to_domain(inlink.host)
-                @internal_inbound_links << inlink.to_s
-            else
-                @external_inbound_links << inlink.to_s
-            end
+class Mechanize::Page
+    #@invalid_links = Hash.new
+    attr_reader :valid_links,:invalid_links,:outbound_links,:internal_outbound_links,:external_outbound_links
+    public
+    def text
+        return Nokogiri::HTML(body).xpath("//text()").text
+        #return body.gsub(/<\/?[^>]*>/, "")
+    end
+    def keywords
+        meta = search("//meta[@name='keywords']").first
+        return meta.attributes["content"].value.split(',') unless meta.nil?
+    end
+    def description
+        meta = search("//meta[@name='description']").first
+        if meta.nil?
+            return false
         end
-        @internal_inbound_links.uniq!
-        @external_inbound_links.uniq!
+        return meta.attributes['content'].value
     end
-    def scan_outbound_links
-        @outbound_links.each do |outlink|
-            w = Webpage.new(outlink)
-            unless w.successful
-                @invalid_links << outlink
-                next
-            end
-            next if w.links.nil?
-            w.links.each do |uri|
-                #uri = URI.parse(uri)
-                #next if uri.host.nil?
-                if Webpage.host_to_domain(uri) == @domain
-                    @back_links << uri.to_s
-                else
-                    @broken_outbound_links << uri.to_s
-                end
-            end
-        end
-        @back_links.uniq!
-        @broken_outbound_links.uniq!
+    def pagerank
+        require 'page_rankr'
+        @pagerank = PageRankr.ranks(@uri.to_s, :google)[:google]
+        return @pagerank
     end
     def scan_links
-        return unless @links.empty?
+        @external_outbound_links = Array.new
+        @internal_outbound_links = Array.new
+        @valid_links = Array.new
+        @invalid_links = Array.new
+        @nofollowed_links = Array.new
         exts_to_ignored = %w(.exe .jpg .png .gif .msi .pdf .swf) #decide before download the uri
-        @page.links.each do |link|
+        links.each do |link|
             #初步解析
-            begin
-                uri = URI.parse(link.href)
-                href = uri.to_s
-            rescue URI::InvalidURIError => e
-                @invalid_links << href
+=begin
+    uri = URI.parse(link.uri).normalize
+    href = uri.to_s
+rescue URI::InvalidURIError => e
+    pp link
+    puts e
+    @invalid_links << link
+    next
+=end
+            #忽略非http请求
+            if link.uri.respond_to?'scheme' and !link.uri.scheme.nil? and link.uri.scheme != 'http' and link.uri.scheme != 'https'
+                @invalid_links << link#todo 不同链接key重复，无法体现
+                next
+            end
+            #忽略非网页文件,忽略js按钮忽略邮件
+            if !link.href.nil? and exts_to_ignored.include?link.href[-4,4]# or href.start_with?'javascript:' or href.start_with?'mailto:'
+                @invalid_links << link
                 next
             end
-            #忽略非网页文件
-            if exts_to_ignored.include?href[-4,4]
-                @ignored_uris << href
+            #nofollow links
+            if link.rel.include?'nofollow'
+                @nofollowed_links << link
                 next
             end
+            if link.respond_to?'fragment' and link.fragment.empty?
+                @invalid_links << link
+                next
+            end
+            pp link
             #处理相对路径
-            if uri.relative?
-                @relative_paths << href
-                href = @uri.merge(href).to_s
-                @internal_outbound_links << href
+            if !link.uri.nil? and link.uri.relative?
+                @invalid_links << link
+                #puts @uri.merge(link)
+                #link.uri = @uri.merge(link.uri)
+                @internal_outbound_links << link unless link.uri == @uri
+            elsif link.uri.nil?
+                warn "warning: host nil #{link.uri}"
+                next
             else
-                href = Webpage.uri_normalize(href)
-                if Webpage.host_to_domain(uri.host) == @domain
-                    @internal_outbound_links << href
+                if link.uri.to_s.start_with?'/' or @uri.merge(link.uri).domain == @uri.domain
+                    @internal_outbound_links << link
                 else
-                    @external_outbound_links << href
+                    @external_outbound_links << link
                 end
             end
-            @links << href
+            @valid_links << link
+        end
+        @outbound_links = @internal_outbound_links + @external_outbound_links
+        @scanned = true
+    end
+end
+class URI::Generic
+    def absolute?()
+        if @scheme or path.start_with?'/'
+            true
+        else
+            false
+        end
+    end
+    def domain
+        domain = (host.match /\.?([a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+)$/)
+        return domain[1] unless domain.nil?
+        return nil
+    end
+=begin
+    def normalize!
+        if path && path == ''
+            set_path('/')
+        end
+        if scheme && scheme != scheme.downcase
+            set_scheme(self.scheme.downcase)
         end
-        @outbound_links = @internal_outbound_links.uniq! + @external_outbound_links.uniq!
+        if host && host != host.downcase
+            set_host(self.host.downcase)
+        end
+        set_fragment(nil) unless fragment.nil?
     end
+=end
 end
+=begin
+class URI::Parser
+    def parse(uri)
+        scheme, userinfo, host, port, registry, path, opaque, query, fragment = self.split(uri)
+        if scheme && URI.scheme_list.include?(scheme.upcase)
+            URI.scheme_list[scheme.upcase].new(scheme, userinfo, host, port, registry, path, opaque, query, nil, self)
+        else
+            URI::Generic.new(scheme, userinfo, host, port, registry, path, opaque, query, nil, self)
+        end
+    end
+end
+a = Mechanize.new
+w = a.get('http://dict.youdao.com/w/abc/')
+w.scan_links
+pp w.internal_outbound_links
+exit
+w.links.each do |link|
+    puts link.rel
+end
+=end

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webpage
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease:
 platform: ruby
 authors:
@@ -11,14 +11,15 @@ bindir: bin
 cert_chain: []
 date: 2012-04-11 00:00:00.000000000 Z
 dependencies: []
-description: to show seo oriented reports of the webpage,newbie's work, careful
+description: modify Mechanize::Page to show seo oriented reports of the webpage,newbie's
+  work, careful
 email: seoaqua@qq.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - webpage.rb
-homepage: http://seoaqua.com
+homepage: http://github.com/seoaqua/ruby-webpage
 licenses: []
 post_install_message:
 rdoc_options: []
@@ -41,5 +42,6 @@ rubyforge_project:
 rubygems_version: 1.8.21
 signing_key:
 specification_version: 3
-summary: to show seo oriented reports of the webpage,newbie's work, careful
+summary: modify Mechanize::Page to show seo oriented reports of the webpage,newbie's
+  work, careful
 test_files: []