free_spider 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c449eb2916a732e30c4720816338b5588d237859
-  data.tar.gz: 291cee72295e6cb010214f3f1ba1c992ed86233d
+  metadata.gz: 77aa733a35759e6c95a5b46afe179525b3647527
+  data.tar.gz: 23811d1a62d030ff1f43a4954123a3e785827227
 SHA512:
-  metadata.gz: c7ada15a4fdb0a3e6ae4e9e76951bb418914cb5fe1145a5f9d87e18f2afb25094c5c07a392eadaafe4988e25cf64838a6bb1613702c5a2bc88ba54a5063f38a5
-  data.tar.gz: 82f77838e76ac8a0e0eef7511e67b91dbc1508793cbbaab169b7a6b256c28c74f79f9ff7d28c7f00541076276bf02e99197e35d5a7bc545c9e366dc28f672421
+  metadata.gz: cbc19ec7685f189514c8ddee406d2ad0ebca43ae8902efe1702481572d5cd324cf7160ef0f49074e79986e5e54dc06f7e72638653aa38eea600c78e236fde708
+  data.tar.gz: fc6322bc17701135206e57da6cf1fabaf91c742cb850b17f89b3629da40a0f03a23813d89646adc1883c65cb310ee88193484b2caff901c5ac88dfb08bdd1bbe

data/lib/free_spider/begin.rb

@@ -1,21 +1,29 @@
 # encoding = utf-8
+
 # 生成gem
 # gem build free_spider.gemspec
 # 安装gem
 # gem install free_spider
+# 提交gem
+# gem push free_spider-0.0.1.gem
+
 # 程序入口
 # require 'free_spider'
 # spider = FreeSpider::Begin.new
 # spider.plan do
-#   site 'http://www.dfrobot.com.cn/'
+#   site 'http://oszine.com/'
 # end
 # spider.crawl
-#调试
+
+# 调试
 # pry -Ilib -rfree_spider
 # irb -Ilib -rfree_spider
 
+# coending = utf-8
 require 'open-uri'
 require 'nokogiri'
+# require 'active_record'
+# require 'mysql2'
 # require 'logger'
 
 module FreeSpider
@@ -26,7 +34,10 @@ module FreeSpider
       @todo = []
       # 已经访问过的链接
       @visited = []
-      @titles = []
+      # 暂时存放内容
+      @news_teaching_content = {}
+      # 文章题目(判断是否重复)
+      @title_saved = []
     end
 
     # 程序制定函数，用户选择需要抓取的网页内容
@@ -40,7 +51,7 @@ module FreeSpider
 
     # 查找网页中的链接
     def find_link(path)
-      p "find_link-------------------"
+      puts "--------find_link--------"
       begin
         crawl if path == nil
         html = open(path).read
@@ -51,23 +62,52 @@ module FreeSpider
         # p @visited
         # p path
         doc = Nokogiri::HTML(html)
-        # 抓取主要内容
+        # 抓取链接加入爬取队列
         doc.css("a").map do |href|
           # 选取内容
-          title = href.attributes["title"]
-          title_content = href.attributes["title"].value unless title.nil?
+          # title = href.attributes["title"]
+          # title_content = href.attributes["title"].value unless title.nil?
           # 处理链接
           href = href.attributes["href"].value unless href.attributes["href"].nil?
+          # 去除重复链接
           href = @site + href unless href.include?("#{@site}")
+
+          # 加入爬取队列
           @todo << href
-          @titles << title_content
         end
+
+        # 抓取主要内容
+        unless doc.at_css(".entry-content").nil?
+          entry_title = doc.css(".entry-title").children.to_html
+          unless @title_saved.include?(entry_title)
+            @title_saved << entry_title
+            content = doc.css(".entry-content").children.to_html
+            @news_teaching_content = {title: entry_title, content: content}
+
+            # # 文章题目
+            # doc.css(".entry-title").each do |entry_title|
+            #   title = entry_title.children.to_html unless entry_title.nil?
+            #   news_teaching_content_tmp = {title: title}
+            # end
+            # # 放入将存入的内容
+            # doc.css(".entry-content").each do |entry_content|
+            #   content = entry_content.children.to_html unless entry_content.nil?
+            #   news_teaching_content_tmp.merge!({content: content})
+            # end
+            # p "--------news_entry--------"
+            # p news_teaching_content_tmp
+            # @news_teaching_content = news_teaching_content_tmp
+          end
+        end
+
+
         # 去除重复链接
-        @todo.uniq
-        # 打印信息, 写入文件
-        puts "#{@visited}"
-        p @titles.uniq.compact
-        write_results_to_file('title_out')
+        # @todo.uniq
+        # 打印信息, 写入文件or数据库
+        # puts "#{@visited}"
+        # p @titles.uniq.compact
+        write_results_to_database
+        # write_results_to_file('title_out')
         crawl
       rescue OpenURI::HTTPError
         puts "404"
@@ -106,7 +146,7 @@ module FreeSpider
 
     # 需要爬取的网站首页
     def site(url)
-      p "-----------------"
+      puts "--------Ready---------"
       if url.empty?
         puts "URL is blank"
       else
@@ -115,10 +155,21 @@ module FreeSpider
       end
     end
 
-    def post_title
-      @titles.uniq.compact
+    # 写入mysql
+    def write_results_to_database
+      news_teaching = FreeSpider::Downloader::NewsTeaching.new(@news_teaching_content)
+      if news_teaching.save
+        puts "--------save success!--------"
+      else
+        puts "--------save error!--------"
+      end
     end
 
+    # def post_title
+    #   @titles.uniq.compact
+    # end
+
+    # 写入文件
     def write_results_to_file(file_name)
       if File.exist?(file_name) || File.new(file_name, "w")
         File.open(file_name, "w") do |f|

data/lib/free_spider/downloader.rb

@@ -1,7 +1,36 @@
 # 下载器的主要职责是抓取网页并将网页内容返还给蜘蛛(Spiders)
 
+require 'active_record'
+require 'mysql2'
+
 module FreeSpider
   module Downloader
+  	ActiveRecord::Base.logger = Logger.new(STDERR)
+
+		# 链接数据库
+		puts "----database_connection-----"
+  	ActiveRecord::Base.establish_connection(
+      adapter: 'mysql2',
+      host: 'localhost',
+      database: 'chuangkejiazu',
+      username: 'root',
+      password: '123'
+    )
+
+  	# 创建表结构
+  	puts "----table_create-----"
+    ActiveRecord::Schema.define do
+		  unless ActiveRecord::Base.connection.tables.include? 'news_teachings'
+		    create_table :news_teachings do |table|
+		      table.column :title,     :string
+		      table.column :content,   :text
+		    end
+		  end
+		end
+
+  	class NewsTeaching < ActiveRecord::Base
+  		validates_presence_of :title, :content
+  	end
 
   end
 end

data/lib/free_spider/version.rb

@@ -1,3 +1,3 @@
 module FreeSpider
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: free_spider
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - free
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-08 00:00:00.000000000 Z
+date: 2015-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.13
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.13
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement