news_crawler 0.0.4 → 1.0.0.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be6f510fc6e9737a50f42f5a6f11f2c21b5cfcb4
-  data.tar.gz: add012faa2d72f7ec93b28df4b55faeea52dd86d
+  metadata.gz: 2ec4bfc6924849f911dfcc86f41fc9e3bb8c68a8
+  data.tar.gz: 292da2b3e3ae3836d2511b2f86687433039b7f87
 SHA512:
-  metadata.gz: 5498cf91cf3c169062edfdc9cdeff42adf4347cdb5addd5a36146e324bcfa96fb40806436034b59decc9730bd9b6ff7f7fbed8055980061f1b637bcb1d20ad82
-  data.tar.gz: 6d20e4c7121fa06898112862b467838f8008837b01e795241cbb744de43eb3e1f147c70ccdde94bbe4745363652041de91ed4341b55b36bcdc7f60b9c01cfca1
+  metadata.gz: 6eb68c98a026a0abb8fe3d1dd4a535ea5a3b4c17bc4ed52543c262ae440604b3515b80688ba06df53eb1e1cf9ff9228ad5fd92c3a55ba478291e7d7fa0fbdb16
+  data.tar.gz: 9ee6ad58b747ab1d4d9eee4b100cd506c8d0a3b751163c657203da8e4773320da915d0eca9583a5f29b91df0ef26f9696132e35cccd7701db28302e5900b0e0d

data/bin/news_crawler

@@ -57,7 +57,7 @@ OptionParser.new do | opts |
   end
 
   opts.on('-t', "--time-out TIME", OptionParser::DecimalInteger,
-          "Wait time (in sec) before stop crawl (crawler is\'nt stopped immediately but terminated gracefully)", 
+          "Wait time (in sec) before stop crawl",
           "If time out isn't specified you can stop crawler by press Ctrl-C") do | t |
     options[:time_out] = t
   end
@@ -97,18 +97,16 @@ if options[:time_out]
   sleep(options[:time_out])
 else
   stop = false
-  while(!stop)
+  while(not stop)
     Signal.trap("INT") do | signo |
       stop = true
     end
   end
 end
 
-puts "Stoping SDS"
-se.graceful_terminate
-se.terminate
-puts "SDS stopped"
 puts "Stoping Downloader"
-dwl.graceful_terminate
 dwl.terminate
 puts "Downloader stopped"
+puts "Stoping SDS"
+se.terminate
+puts "SDS stopped"

data/lib/news_crawler/downloader.rb

@@ -42,18 +42,22 @@ module NewsCrawler
     # @param [ NewsCrawler::URLQueue ] queue url queue
     def initialize(start_on_create = true, queue = NewsCrawler::Storage::URLQueue, **opts)
       @queue = queue
-      @urls = queue.find_unvisited
       @concurrent_download = opts[:concurrent] || CONCURRENT_DOWNLOAD
       @wait_time = 1
-      @status = :running
       @stoping = false
+      get_new_url
       wait_for_url if start_on_create
     end
 
     # Start downloader with current queue
     # URL successed fetch is marked and result's stored in DB
     def run
-      @status = :running
+      wait_for_url
+    end
+
+    private
+    # Download urls are given (in @urls)
+    def download
       hydra = Typhoeus::Hydra.new(max_concurrency: @concurrent_download)
       # TODO Log here
       @urls = @urls.keep_if do | url |
@@ -73,40 +77,40 @@ module NewsCrawler
         re
       end
       hydra.run
-      @urls = []
-      wait_for_url
-    end
-
-    # Graceful terminate this downloader
-    def graceful_terminate
-      @stoping = true
-      while @status == :running
-        sleep(1)
-      end
     end
 
-    private
     # Waiting for new urls're added to queue, using backoff algorithms
+    # Invoke download when suitable
     def wait_for_url
-      @status = :waiting
-      if @stoping # check for stop flag
-        return
+      while not @stoping do
+        if @queuing_urls.size == 0
+          get_new_url
+        end
+        if @queuing_urls.size == 0
+          backoff_sleep
+        else
+          if @stoping
+            return
+          end
+          @wait_time = 1
+          @urls = @queuing_urls.shift(@concurrent_download * 2)
+          download
+          sleep 0.01 # delay to receive terminate signal
+        end
       end
+    end
+
+    # Sleep using backoff algorithm
+    # @params [ Fixnum ] seconds
+    def backoff_sleep
       sleep @wait_time
-      get_new_url
-      if @urls.size == 0
-        if @wait_time < 30
-          @wait_time = @wait_time * 2
-        end
-        wait_for_url
-      else
-        @wait_time = 1
-        run
+      if @wait_time * 2 <= 4
+        @wait_time = @wait_time * 2
       end
     end
 
     def get_new_url
-      @urls = @queue.find_unvisited
+      @queuing_urls = @queue.find_unvisited
     end
   end
 end

data/lib/news_crawler/link_selector/same_domain_selector.rb

@@ -87,6 +87,7 @@ module NewsCrawler
         }
       end
 
+      # run selector
       def run
         @status = :running
         return if @stoping
@@ -94,7 +95,7 @@ module NewsCrawler
           @status = :stopped
           return
         end
-        while !@stoping
+        while true do
           url = next_unprocessed(@max_depth - 1)
           while (url.nil?)
             wait_for_url
@@ -103,6 +104,7 @@ module NewsCrawler
           NCLogger.get_logger.info "Processing #{url}"
           extract_url(url)
           mark_processed(url)
+          sleep 0.01 # delay to receive terminate signal
         end
       end
 
@@ -154,20 +156,12 @@ module NewsCrawler
         return false
       end
 
-      # Graceful terminate this selector
-      def graceful_terminate
-        @stoping = true
-        while @status == :running
-          sleep(1)
-        end
-      end
-
       private
       # Waiting for new urls're added to queue, using backoff algorithms
       def wait_for_url
         @status = :waiting
         sleep @wait_time
-        if @wait_time < 30
+        if @wait_time < 4
           @wait_times = @wait_time * 2
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: news_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 1.0.0.pre.1
 platform: ruby
 authors:
 - Hà Quang Dương
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-12 00:00:00.000000000 Z
+date: 2013-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -195,9 +195,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.0.3