meiriyigua 0.0.1 → 0.0.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    N2VhZDc2ODBjYTliYzkxMTliNzQyNDYyNmZiZGI3NGM3YTJkZmM3MA==
-  data.tar.gz: !binary |-
-    YmZlZjE0YTE4N2U2ODY1ZDdkNTAwOTRiNjE5YjMyMTMwZWJlMjU1Mg==
+SHA1:
+  metadata.gz: 8e5647d5d4245de1cef59d3ce7f3b017f34680e8
+  data.tar.gz: 0afca96a706517437f26a50e3a1bc440164dbc05
 SHA512:
-  metadata.gz: !binary |-
-    YWE3NmNlYmIwYjExZTA3MDIxOWRmMTc4NDMxYWEwMmJmNDEzNTE3OTcxYzMy
-    ZGViMjNiNGZlZDkyYTBhZDBkZWJkNTgxNGNiYzkyNDMxN2UxZTg5ZmMzODgx
-    NDdjYWFmYmYzZjlkYTE4MGExNDQ2N2ZlMzYzNzdhMTRiNjRmZGE=
-  data.tar.gz: !binary |-
-    ZmI0MTQ2MzJmZDgyOTcyNzRlMGEwNDMwNDQ5OWMwNDRhOTZiMTA0MWM4NTg3
-    ODk4NWJmMmVlMzJjMWNkODMxMDZjZmJlZGRkNzg0ZWJkNWJmNGVjNDNmOGFk
-    MmMxMjdkNTk1NjFmYWQwYmVlZDBjYWIwNTcwNTk1OTQwNGJjN2U=
+  metadata.gz: 81a747d5cdb187c3fbdb2c69eb7fbbad72f4b35735d72b488e0ec1a0b4518c2d9fc30b157e2efc86a9e238c9195a7ee37f1880fc752b860203cb831881e7b121
+  data.tar.gz: bfd076a7a1dc43d57df1216f6e1b834670375c9ac0ffe49bee5f087da3064da4b15984b46445fc01c1452a98103542d0e571aa8d7f2881dae471524e2f175995

data/lib/meiriyigua/baidu_crawl.rb

@@ -35,7 +35,10 @@ module Meiriyigua
       search_form = page.form_with(:name => "f1")
       search_form.field_with(:name => "wd").value = title
       search_results = @agent.submit search_form
-      search_results.search('div.c-container:first-of-type div.c-abstract').text
+
+      result = ""
+      search_results.search('div.c-container div.c-abstract').each_with_index{|a, i| result << "提示#{i+1}\r\n #{a.text}\r\n\r\n"}
+      result
     end
   end
 end

data/lib/meiriyigua/models.rb

@@ -1,7 +1,7 @@
 # -*- encoding : utf-8 -*-
 require 'data_mapper'
 
-#DataMapper::Logger.new($stdout, :debug)
+DataMapper::Logger.new($stdout, :debug)
 DataMapper::Model.raise_on_save_failure = true
 DataMapper.setup(:default, "sqlite://#{Dir.pwd}/data.db")
 

data/lib/meiriyigua/version.rb

@@ -1,4 +1,4 @@
 # -*- encoding : utf-8 -*-
 module Meiriyigua
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/meiriyigua.rb

@@ -5,11 +5,16 @@ require "meiriyigua/list_crawl"
 require "meiriyigua/detail_crawl"
 require "meiriyigua/baidu_crawl"
 require "meiriyigua/post_client"
+require "meiriyigua/models"
 
 module Meiriyigua
   def run
     while true
       begin
+        ((Meiriyigua::Models::UrlRecord.all(baidu_at: nil) | Meiriyigua::Models::UrlRecord.all(publish_at: nil)) & Meiriyigua::Models::UrlRecord.all(:detail_at.lt => (Date.today - 7))).each do |url_record|
+          url_record.page_record.destroy
+          url_record.destroy
+        end
 
         threads = []
         threads << Thread.new do
@@ -32,7 +37,7 @@ module Meiriyigua
         threads.each { |thr| thr.join }
 
       rescue
-        puts "出错了 #{$!.message}"
+        puts "出错了 #{$!.message}\n#{$!.backtrace.join("\n")}"
       end
       sleep 60*10
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: meiriyigua
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - mangege
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-07 00:00:00.000000000 Z
+date: 2014-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -84,14 +84,14 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: nil
@@ -127,18 +127,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.1.3
 signing_key: 
 specification_version: 4
 summary: nil
 test_files: []
+has_rdoc: