links_crawler 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c07bea8f84cb2543c550fb14f3d8482d7a960918
+  data.tar.gz: 11e05b8107486dfa22b004c97b867bd938ffffa7
+SHA512:
+  metadata.gz: d0bceaa69b62924ff60f1cb180d6b56b4c1df58b74fee4f635302471cc4fb6b6d35e90cbde94ad9c1af80e101c9bfe3cadd823aa8b97ac9cb83e240f6fbb4c39
+  data.tar.gz: 7a012812983be0597d5787547fc75ed4a6b63e63955dd13de5b51f92ca8c9b904224766e0751132e193fe038669f6a2dd68192aaeab739136130dfd1db5a94e7

data/lib/links_crawler.rb

@@ -0,0 +1,24 @@
+require 'net/http'
+
+module LinksCrawler #products list, feedbacks list
+  def self.included(recipient)
+    recipient.class_eval do
+      include ModelInstanceMethods
+    end
+  end
+
+  module ModelInstanceMethods   
+    def import(target, importer)
+      case importer.origin
+      when "eBay"
+        extend Ebay::Importer
+        target_at_and_for target, importer
+      when "iOffer"
+        extend Ioffer::Importer
+        target_at_and_for target, importer
+      end 
+    end
+  end# instance methods
+end
+
+

data/lib/links_crawler/model_instance_methods.rb

@@ -0,0 +1,72 @@
+require 'net/http'
+
+module LinksCrawler
+  module ModelInstanceMethods
+
+    def fetch_links(dname,path)
+      uri = "http://#{dname}#{path}" unless dname.include?("http")
+      uri = URI(uri) 
+
+      Net::HTTP.start(uri.host, uri.port) do |http|
+        request = Net::HTTP::Get.new uri
+        res = http.request request
+        str_body = res.body                                 # proper argument
+
+        relative_links = str_body.scan(/href="[^(http|\.\.|#)].*?"/)  
+          relative_links_path = relative_links.collect do |relative_link|
+          relative_link.delete("\"")[5..-1]
+        end
+
+        abs_links = str_body.scan(/href="http.*?"/).select do |href|
+          href.include? dname
+        end
+        abs_links_path = abs_links.collect do |abs_href|
+          URI(abs_href.delete("\"")[5..-1]).path
+        end
+
+        mixed_links = (relative_links_path + abs_links_path).uniq
+        mixed_links.each do |k|
+          tmp = k  #.delete("\"")[5..-1]
+          if path == '/'
+            @arr_links << tmp
+          else
+            @arr_links.unshift tmp if is_valid_path?(tmp)
+          end
+        end
+      end
+    end
+
+    def is_valid_path?(tmp)
+      !@arr_links.include?(tmp) && !tmp.include?("javascript") && !tmp.match(/css$/)
+    end
+
+    def traverse(dname) 
+      root = '/'
+      fetch_links(dname,root)
+      #debugger
+      while link = @arr_links.shift
+        if !@traversed.include? link
+          @traversed << link
+          puts "dname #{dname} link #{link}"
+          begin 
+            fetch_links(dname,link)
+            @arr_links.delete(link)    # remove the link that having been traversed
+          rescue 
+            @arr_links.delete(link)    # remove the link that having been traversed
+          end
+        end
+      end
+    end
+
+
+    def search(dname)
+      @arr_links = []
+      @traversed = []
+      traverse(dname)
+      @traversed.each do  |i|
+        puts "http://#{dname}" + i
+      end
+    end
+
+  end
+end

data/lib/links_crawler/version.rb

@@ -0,0 +1,3 @@
+module LinksCrawler
+  VERSION="0.0.1"
+end

data/links_crawler.gemspec

@@ -0,0 +1,20 @@
+#-*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "links_crawler/version"
+
+Gem::Specification.new do |s|
+  s.name        = "links_crawler"
+  s.version     = LinksCrawler::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["wenbo"]
+  s.email       = ["yiyun6674@hotmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{check how many links are available inside the website}
+  s.description = <<-EOF
+    check how many links are available inside the website
+    EOF
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test}/*`.split("\n")
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: links_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- wenbo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-02 00:00:00.000000000 Z
+dependencies: []
+description: |2
+      check how many links are available inside the website
+email:
+- yiyun6674@hotmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/links_crawler.rb
+- lib/links_crawler/model_instance_methods.rb
+- lib/links_crawler/version.rb
+- links_crawler.gemspec
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: check how many links are available inside the website
+test_files: []