news_crawler 0.0.0 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c77a9447146f179d2b1d3013874a8ea4377be1c5
-  data.tar.gz: f66b44459c9afc5f962e91bf135827b9014ce196
+  metadata.gz: 50e84e9674b22d98be7b72371513219da5a23d38
+  data.tar.gz: bd6cd50fe658c960134fdfff53631e32e63e4b76
 SHA512:
-  metadata.gz: 84feee86effa11929b40797b93e42a86491a710af20f3801e7ff8becebfa8f2d4f8a465bbe9fff1f4cb9683f49af21aac3df7dd27372cdd1f8d5567147014dd9
-  data.tar.gz: fc6a53bc7be2a01f26e78a9bcd8bd58a55b7cb34509be75ce5a1878a1311c794c7dcc4dd25ebc2697dbef2dc080dd1c57ae39f2ad03e5c7e0c23a5bdff227259
+  metadata.gz: 405419795794f78bf0608d2b66707d842c69c7c53d1b312a2569e9c82e482563ea28d34c5183accd3bf4b213fc6291305d8295f9310918854cd24e19c1cf6a83
+  data.tar.gz: 53e089fb98b6a9e583c54119215ead3444fcc285d71f3bf5ca6f188b55db13a45fa6a87d414631a81dc2f092cedc726d77907d2d542671a721d92581a4876df8

data/bin/news_crawler

@@ -28,6 +28,7 @@ require 'news_crawler/nc_logger'
 
 require 'news_crawler/downloader'
 require 'news_crawler/link_selector/same_domain_selector'
+require 'news_crawler/storage/url_queue'
 
 include NewsCrawler::Storage
 
@@ -54,6 +55,12 @@ OptionParser.new do | opts |
           'Maximum depth of url to crawl') do  | d |
     options[:max_depth] = d
   end
+
+  opts.on('-t', "--time-out TIME", OptionParser::DecimalInteger,
+          "Wait time (in sec) before stop crawl (crawler is\'nt stopped immediately but terminated gracefully)", 
+          "If time out isn't specified you can stop crawler by press Ctrl-C") do | t |
+    options[:time_out] = t
+  end
 end.parse!
 
 
@@ -71,7 +78,11 @@ end
 
 if ARGV.size > 0
   url = ARGV[0]
-  URLQueue.add(url)
+  begin
+    URLQueue.add(url)
+    rescue NewsCrawler::Storage::URLQueue::DuplicateURLError
+    NewsCrawler::NCLogger.get_logger.info("URL existed")
+  end
 end
 
 puts "Starting Downloader"
@@ -81,13 +92,22 @@ dwl.async.run
 puts "Starting SDS"
 se = NewsCrawler::LinkSelector::SameDomainSelector.new(options[:max_depth] || 1, false)
 se.async.run
+
+if options[:time_out]
+  sleep(options[:time_out])
+else
+  stop = false
+  while(!stop)
+    Signal.trap("INT") do | signo |
+      stop = true
+    end
+  end
+end
+
 puts "Stoping SDS"
 se.graceful_terminate
 se.terminate
 puts "SDS stopped"
-
-sleep(5)
-
 puts "Stoping Downloader"
 dwl.graceful_terminate
 dwl.terminate

data/lib/news_crawler/nc_logger.rb

@@ -34,13 +34,13 @@ module NewsCrawler
     end
 
     # Set logger level
-    # param [ Logger::Severity ] l level
+    # @param [ Logger::Severity ] l level
     def self.set_level(l)
       get_logger.level = l
     end
 
     # Set logger, should same API as Ruby Logger
-    # param [ Object ] l logger
+    # @param [ Object ] l logger
     def self.set_logdev(ld)
       @logger = Logger.new(ld)
       @logger.progname = 'news_crawler'

data/lib/news_crawler/storage/raw_data.rb

@@ -48,8 +48,8 @@ module NewsCrawler
         end
 
         # Add entry to raw data collection
-        # param [ String ] url
-        # param [ String ] body
+        # @param [ String ] url
+        # @param [ String ] body
         def add(url, body)
           @engine.add(url, body)
         end

data/lib/news_crawler/storage/raw_data/mongo_storage.rb

@@ -43,8 +43,8 @@ module NewsCrawler
         end
 
         # Add entry to raw data collection, overwrite old data
-        # param [ String ] url
-        # param [ String ] body
+        # @param [ String ] url
+        # @param [ String ] body
         def add(url, body)
           @coll.update({:url   => url},
                        {:$set  => {:body => body}},

data/lib/news_crawler/storage/raw_data/raw_data_engine.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 #--
 # NewsCrawler - a website crawler
 #
@@ -41,8 +42,8 @@ module NewsCrawler
         end
 
         # Add entry to raw data collection
-        # param [ String ] url
-        # param [ String ] body
+        # @param [ String ] url
+        # @param [ String ] body
         def add(url, body)
           raise NotImplementedError
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: news_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.0.2
 platform: ruby
 authors:
 - Hà Quang Dương