RubyGems - webpage - Versions diffs - 0.0.2 → 0.0.3 - Mend

webpage 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (2) hide show

data/webpage.rb +206 -49
metadata +1 -1

data/webpage.rb CHANGED Viewed

@@ -3,25 +3,69 @@ require 'pp'
 require 'mechanize'
 require 'uri'
 class Webpage
-    attr_reader:links
+    attr_reader:links,:successful,:related_uris
     def initialize(uri)
-        @uri = URI.parse(encode(uri))
+        @uri = URI.parse(uri_encode(uri))
         @outbound_links = Array.new
-        @outter_inbound_links = Array.new
-        @inbound_links = Array.new
+        @internal_outbound_links = Array.new
+        @external_outbound_links = Array.new
+        @broken_outbound_links = Array.new
+        @external_inbound_links = Array.new
+        @back_links = Array.new
+        @internal_inbound_links = Array.new
+        @external_inbound_links = Array.new
         @internal_links = Array.new
         @links = Array.new
         @uri_dirname = File.dirname(@uri.path)
         @uri_domain = host_to_domain @uri.host
         @accessed_uri = Array.new
+        @page = ''
+        @related_uris = Array.new
+        @successful = false
+        begin
+            agent = Mechanize.new
+            agent.open_timeout = 5
+            @page = agent.get @uri.to_s
+            @page.body = @page.body.force_encoding(@page.encoding).encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "?")
+            @page.links.each do |link| #1
+                next if link.href.nil?
+                uri = uri_encode(link.href.strip)
+                begin
+                    @links << @uri.merge(uri).to_s
+                rescue URI::InvalidURIError,URI::InvalidComponentError
+                    warn "ignore\nparsed: #{uri} \noriginal: #{link.href}"
+                end
+            end
+            @successful = true
+        rescue Exception => e
+            warn "#{e}:#{@uri}"
+        end
+        @links.uniq!
+        scan_links
     end
-    def encode(str)
-        return URI.encode(str,Regexp.new("[^#{URI::PATTERN::UNRESERVED+'#:/?%&='}]"))
+    def encoding
+        return @page.encoding
     end
-    def host_to_domain(host)
-        return (host.match /\.?([a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+)$/)[1]
+    def keywords
+        return @page.search("//meta[@name='keywords']").first.attributes["content"].value.split(',')
+    end
+    def body
+        return @page.body unless @page.body.include?'<html>'
+        return String.new
+    end
+    def text
+        return body.gsub(/<\/?[^>]*>/, "")
+    end
+    def title
+        return @page.title unless @page.title.nil?
+        return String.new
     end
     #get all links from html content
     #1.$all = get all <a>
     #2.$href = get all href from $all
@@ -34,7 +78,7 @@ class Webpage
             agent.get @uri do |page|
                 page.links.each do |link| #1
                     next if link.href.nil?
-                    uri = encode(link.href.strip)
+                    uri = uri_encode(link.href.strip)
                     begin
                         @links << @uri.merge(uri).to_s
                     rescue URI::InvalidURIError,URI::InvalidComponentError
@@ -54,32 +98,117 @@ class Webpage
             warn "#{e}.#{@uri}"
         end
         return Array.new if @links.empty?
-        @links = @links.uniq - @accessed_uri
-        @accessed_uri += @links
-        @links.each do |a|
-            uri = URI.parse(encode(a))
-            next if uri.host.nil?
-            if uri.host.end_with?@uri_domain
-                @internal_links << a
-            else
-                @outbound_links << a
-            end
-        end
+        #@links = @links.uniq - @accessed_uri
+        #@accessed_uri += @links
+        @links.uniq!
+        puts @links
+        scan_links
         return @links
     end
     def internal_links
-        return @internal_links if links
-        return false
+        return @internal_links unless @internal_links.empty?
+        scan_links
+        return @internal_links
+    end
+    def external_outbound_links
+        return @external_outbound_links  unless @external_outbound_links.empty?
+        links
+        return @external_outbound_links
+    end
+    def internal_outbound_links
+        return @internal_outbound_links unless @internal_outbound_links.empty?
+        links
+        return @internal_outbound_links
     end
     def outbound_links
-        return @outbound_links if links
-        return false
+        return external_outbound_links + internal_outbound_links
+    end
+    def back_links#inbound links among all the outbound links
+        return @back_links unless @back_links.empty?
+        scan_outbound_links
+        return @back_links
+    end
+    def broken_outbound_links
+        return @broken_outbound_links unless @broken_outbound_links.empty?
+        scan_outbound_links
+        return @broken_outbound_links
+    end
+    def external_inbound_links#outter inbound links
+        return @external_inbound_links unless @external_inbound_links.empty?
+        scan_inbound_links
+        return @external_inbound_links
+    end
+    def internal_inbound_links
+        return @internal_inbound_links unless @internal_inbound_links.empty?
+        scan_inbound_links
+        return @internal_inbound_links
     end
-    def inbound_links
-        return @inbound_links unless @inbound_links.empty?
+    def pagerank
+        return @pagerank unless @pagerank
+        require 'PageRankr'
+        @pagerank = PageRankr.ranks(@uri.to_s, :google)
+        return @pagerank
+    end
+    def ppl#pagerank per link
+        return (@pagerank / links.count)
+    end
+    def related_uris(related_keywords=Array.new,seed_uris=Array.new,checked_uris=Array.new,related=Array.new)
+        raise "related_keywords is not array,but a #{related_keywords.class}" unless related_keywords.class == Array and seed_uris.class == Array and checked_uris.class == Array
+        related_keywords.concat(keywords).uniq!
+        seed_uris.concat(external_outbound_links).uniq!
+        while seed_uris.size>0
+            uri = seed_uris.first
+            checked_uris << uri unless checked_uris.include?uri
+            seed_uris.delete(uri)
+            w = Webpage.new uri
+            next unless w.successful
+            text = w.body + w.title
+            related_keywords.each do |word|
+                if text.include?word
+                    related << uri
+                    seed_uris.concat(w.external_outbound_links).uniq!
+                    break
+                end
+            end
+        end
+        return related
+    end
+    private
+    def scan_links
+        @links.each do |a|
+            begin
+                uri = URI.parse(uri_encode(a))
+            rescue URI::InvalidURIError =>e
+                puts "#{e}:#{uri}"
+                next
+            end
+            next if uri.host.nil?
+            if uri.host.end_with?@uri_domain
+                @internal_links << a
+            elsif uri.scheme.start_with?'http'
+                if host_to_domain(uri.host) == @uri_domain
+                    @internal_outbound_links << uri.to_s
+                else
+                    @external_outbound_links << uri.to_s
+                end
+                #@outbound_links << a
+            end
+        end
+        @back_links.uniq!
+        #@outbound_links.uniq!
+        @internal_outbound_links.uniq!
+        @external_outbound_links.uniq!
+    end
+    def scan_outbound_links
         outbound_links.each do |outlink|
             begin
                 w = Webpage.new(outlink)
@@ -87,42 +216,70 @@ class Webpage
                 warn "bad uri:#{outlink}"
                 next
             end
+            next if w.links.nil?
             w.links.each do |uri|
                 next unless uri.start_with?'http'
                 begin
-                    uri = URI.parse(encode(uri))
+                    uri = URI.parse(uri_encode(uri))
                     next if uri.host.nil?
-                    @inbound_links << uri.to_s if uri.host.end_with?@uri_domain
+                    if uri.host.end_with?@uri_domain
+                        @back_links << uri.to_s
+                    else
+                        @broken_outbound_links << uri.to_s
+                    end
                 rescue URI::InvalidURIError
                     warn "bad uri:#{uri}"
                 end
             end
         end
-        return @inbound_links.uniq
+        @back_links.uniq!
+        @broken_outbound_links.uniq!
     end
-    def outter_inbound_links
-        return @outter_inbound_links unless @outter_inbound_links.empty?
-        inbound_links.each do |inlink|
+    def scan_inbound_links
+        back_links.each do |inlink|
             inlink = URI.parse inlink
-            @outter_inbound_links << inlink.to_s unless @uri_domain == host_to_domain(inlink.host)
+            if @uri_domain == host_to_domain(inlink.host)
+                @internal_inbound_links << inlink.to_s
+            else
+                @external_inbound_links << inlink.to_s
+            end
         end
-        return @outter_inbound_links
+        @internal_inbound_links.uniq!
+        @external_inbound_links.uniq!
     end
-    def friend_links#inbound && outbound
+    def uri_encode(str)
+        return URI.encode(str,Regexp.new("[^#{URI::PATTERN::UNRESERVED+'#:/?%&='}]"))
     end
-    def pagerank
-        return @pagerank unless @pagerank
-        require 'PageRankr'
-        @pagerank = PageRankr.ranks(@uri.to_s, :google)
-        return @pagerank
+    def host_to_domain(host)
+        domain = (host.match /\.?([a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+)$/)
+        return domain[1] unless domain.nil?
+        return false
     end
+end
-    def ppl#pagerank_per_link
-        return (@pagerank / links.count)
-    end
+w = Webpage.new('http://cidian.youdao.com')
+#puts w.external_outbound_links
+related_keywords = %w(词典 辞典 辞海 译 英语 法语 日语 韩语 语言)
+puts w.related_uris(related_keywords)
+exit
+require 'yaml'
+filename = './cidian.yaml'
+if File.exists?(filename)
+    cached_cidian = YAML.load(File.read(filename))
+else
+    cached_cidian = Hash.new
+    cached_cidian[:seed_uris] = Array.new
+    cached_cidian[:checked_uris] = Array.new
+    cached_cidian[:related_keywords] = %w(词典 辞典 辞海 译 英语 法语 日语 韩语 语言)
+    cached_cidian[:related_uris] = Array.new
+end
+at_exit do
+    File.open(filename,'w'){|f|f.puts(cached_cidian.to_yaml)}
 end
-w = Webpage.new('http://auto.163.com')
-puts w.outter_inbound_links
+#puts w.related_uris(cached_cidian[:seed_uris],cached_cidian[:related_keywords],cached_cidian[:checked_uris],cached_cidian[:related_uris])
+puts w.external_inbound_links
+puts w.internal_inbound_links
+puts w.pagerank
+puts w.external_inbound_links

metadata CHANGED Viewed

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webpage
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease:
 platform: ruby
 authors: