tjcrawler 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d862225dcccd8804520d59ed284c9e536eaed0c8
-  data.tar.gz: 5ce47b642a5bae952139f8ad53adb00d4f697e49
+  metadata.gz: 4b2e68041b0126fe587a4348cb7b5586231d025b
+  data.tar.gz: 827c4d9d45cdb1c4fd33cabacdc5c10af36def81
 SHA512:
-  metadata.gz: b376fdb73d63fe698ddd1dbb787ab3e5973a58fdc1e5c4514090c0785e3cf594dbbb3398d8b26dd555f255e899a70b1c3406aa741fe1af7273dae1913f8265e3
-  data.tar.gz: da0abd28dfb2245fd5b9a54d3e3094d4db1c8add13385eed00b13f15580d877a0ff22fd41e49c04e6c8361dead88f23edfd44d3c60e1592cad9bfb63f549c7ff
+  metadata.gz: 0be2fdd12d3a5079e6979d30f200135d1eff5c193541db58e71b8a289e2d334d9b49eaad3070d0a3df4c38d20a5049d9c7ea99b062ed6d0be7d492732e5aa3c6
+  data.tar.gz: f9d0fa93536ebac50b13dad8988770b7bb1d97b12170d29c8ed53461c33d35aa5dde4753a4b58f379a7e44b735fc9773e482cf235e484984b6c75511985b5a6c

data/lib/tjcrawler/crawler.rb

@@ -22,7 +22,8 @@ module Tjcrawler
         sleep 1 until page = Page.dequeue
         print :'.'
         result = crawl page.url
-        page.touch(:crawled_at) if page.update(content: result.content)
+        page.update(content: result.content)
+        page.touch(:crawled_at)
         result.links.each{ |url| Page.enqueue url }
       end
     end

data/lib/tjcrawler/parser.rb

@@ -1,10 +1,9 @@
 require 'nokogiri'
 require 'tjcrawler/page'
-
 module Tjcrawler
   class Parser
-    # a nokogiri doc will be yield in block, return true/flase
-    # for successful/failed parsing.
+    @@semaphore = Mutex.new
+    # a nokogiri doc will be yield in block
     def initialize &block
       yield 'Block required' unless block_given?
       @strategy = block
@@ -12,7 +11,8 @@ module Tjcrawler
 
     def parse content
       doc = Nokogiri::HTML(content)
-      ret = @strategy[doc]
+      ret = nil
+      @@semaphore.synchronize{ ret = @strategy[doc] }
       print :'.'
       ret
     end
@@ -20,7 +20,8 @@ module Tjcrawler
     def start
       loop do
         sleep 1 until page = find_next
-        page.touch(:parsed_at) if parse(page.content)
+        parse(page.content)
+        page.touch(:parsed_at)
       end
     end
 

data/lib/tjcrawler/version.rb

@@ -1,3 +1,3 @@
 module Tjcrawler
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tjcrawler
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Jian Weihang