free_spider 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c449eb2916a732e30c4720816338b5588d237859
+  data.tar.gz: 291cee72295e6cb010214f3f1ba1c992ed86233d
+SHA512:
+  metadata.gz: c7ada15a4fdb0a3e6ae4e9e76951bb418914cb5fe1145a5f9d87e18f2afb25094c5c07a392eadaafe4988e25cf64838a6bb1613702c5a2bc88ba54a5063f38a5
+  data.tar.gz: 82f77838e76ac8a0e0eef7511e67b91dbc1508793cbbaab169b7a6b256c28c74f79f9ff7d28c7f00541076276bf02e99197e35d5a7bc545c9e366dc28f672421

data/lib/free_spider/begin.rb

@@ -0,0 +1,131 @@
+# encoding = utf-8
+# 生成gem
+# gem build free_spider.gemspec
+# 安装gem
+# gem install free_spider
+# 程序入口
+# require 'free_spider'
+# spider = FreeSpider::Begin.new
+# spider.plan do
+#   site 'http://www.dfrobot.com.cn/'
+# end
+# spider.crawl
+#调试
+# pry -Ilib -rfree_spider
+# irb -Ilib -rfree_spider
+
+require 'open-uri'
+require 'nokogiri'
+# require 'logger'
+
+module FreeSpider
+  class Begin
+
+    def initialize
+      # 找到的链接
+      @todo = []
+      # 已经访问过的链接
+      @visited = []
+      @titles = []
+    end
+
+    # 程序制定函数，用户选择需要抓取的网页内容
+    def plan(&block)
+      if block_given?
+        instance_eval(&block)
+      else
+        puts "no plan"
+      end
+    end
+
+    # 查找网页中的链接
+    def find_link(path)
+      p "find_link-------------------"
+      begin
+        crawl if path == nil
+        html = open(path).read
+        # html = open('http://www.dfrobot.com.cn/').read
+        # 访问过的链接放入数组
+        @visited << path
+        # p "================"
+        # p @visited
+        # p path
+        doc = Nokogiri::HTML(html)
+        # 抓取主要内容
+        doc.css("a").map do |href|
+          # 选取内容
+          title = href.attributes["title"]
+          title_content = href.attributes["title"].value unless title.nil?
+          # 处理链接
+          href = href.attributes["href"].value unless href.attributes["href"].nil?
+          href = @site + href unless href.include?("#{@site}")
+          @todo << href
+          @titles << title_content
+        end
+        # 去除重复链接
+        @todo.uniq
+        # 打印信息, 写入文件
+        puts "#{@visited}"
+        p @titles.uniq.compact
+        write_results_to_file('title_out')
+        crawl
+      rescue OpenURI::HTTPError
+        puts "404"
+        crawl
+      rescue RuntimeError
+        puts "redirection forbidden"
+        crawl
+      rescue URI::InvalidURIError
+        puts "bad URI"
+        crawl
+      ensure
+
+      end
+    end
+
+    # 程序开始函数
+    def crawl
+      path = nil
+      loop do
+        # 选取找到的链接中的一个链接
+        path = @todo.shift
+        break if path.nil?
+        # 如果是访问过的链接就重新选取
+        break unless @visited.include?(path)
+        # 去掉外部链接
+        # 去掉特殊链接
+      end
+      if path.nil?
+        puts "结束"
+        # 输出抓取内容
+        # post_title
+        return
+      end
+      find_link(path)
+    end
+
+    # 需要爬取的网站首页
+    def site(url)
+      p "-----------------"
+      if url.empty?
+        puts "URL is blank"
+      else
+        @site = url
+        @todo << @site
+      end
+    end
+
+    def post_title
+      @titles.uniq.compact
+    end
+
+    def write_results_to_file(file_name)
+      if File.exist?(file_name) || File.new(file_name, "w")
+        File.open(file_name, "w") do |f|
+          f.write(@titles.uniq.compact)
+        end
+      end
+    end
+
+  end
+end

data/lib/free_spider/downloader.rb

@@ -0,0 +1,7 @@
+# 下载器的主要职责是抓取网页并将网页内容返还给蜘蛛(Spiders)
+
+module FreeSpider
+  module Downloader
+
+  end
+end

data/lib/free_spider/logger.rb

@@ -0,0 +1,7 @@
+# 打印爬取过程中出现的错误
+
+module FreeSpider
+  module Logger
+
+  end
+end

data/lib/free_spider/scheduler.rb

@@ -0,0 +1,7 @@
+# 调度程序从Scrapy引擎接受请求并排序列入队列，并在Scrapy引擎发出请求后返还给他们
+
+module FreeSpider
+  module Scheduler
+
+  end
+end

data/lib/free_spider/spiders.rb

@@ -0,0 +1,25 @@
+# 蜘蛛是有Scrapy用户自己定义用来解析网页并抓取制定URL返回的内容的类，每个蜘蛛都能处理一个域名或一组域名。
+# 用来定义特定网站的抓取和解析规则。
+
+module FreeSpider
+  module Spiders
+    class Parser
+
+      def initialize(url, css, attributes)
+        @url = url
+        @css = css
+        @attributes = attributes
+      end
+
+      def fetcher
+        html = open(@url).read
+        doc = Nokogiri::HTML(html)
+        # 找到网页中需要的内容
+        doc.css("#{@css}").map do |href|
+          href.attributes["#{@attributes}"].value
+        end
+      end
+    end
+
+  end
+end

data/lib/free_spider/storage.rb

@@ -0,0 +1,7 @@
+# 存储得到的数据
+
+module FreeSpider
+  module Storage
+
+  end
+end

data/lib/free_spider/version.rb

@@ -0,0 +1,3 @@
+module FreeSpider
+  VERSION = "0.0.1"
+end

data/lib/free_spider.rb

@@ -0,0 +1,7 @@
+require "free_spider/version"
+require "free_spider/downloader"
+require "free_spider/spiders"
+require "free_spider/logger"
+require "free_spider/scheduler"
+require "free_spider/storage"
+require "free_spider/begin"

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: free_spider
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- free
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 10.1.0
+description: A simple spider
+email:
+- 747549945@qq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/free_spider.rb
+- lib/free_spider/begin.rb
+- lib/free_spider/downloader.rb
+- lib/free_spider/logger.rb
+- lib/free_spider/scheduler.rb
+- lib/free_spider/spiders.rb
+- lib/free_spider/storage.rb
+- lib/free_spider/version.rb
+homepage: https://github.com/free1/free_spider
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: A simple spider
+test_files: []