news_crawler 0.0.3.pre.1 → 0.0.3.pre.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e52eaf3135e05aba597c96a7e9cafa43dba73cc8
-  data.tar.gz: 7e50f113f2b5be1f4932eb7ba83975e216777c89
+  metadata.gz: 65491e340a0cb59c0ecac8f6a78d2dd0405fce65
+  data.tar.gz: 44a9cc62f420914287f42856928db6a3ff9714c9
 SHA512:
-  metadata.gz: 1ce7400c42047ad78954b0e4f45e1493fb7833926e69864a0814662e5ddac4f6423247083fe943fbab33085ad27c410dbb500326690c74290821bf3b24b2454a
-  data.tar.gz: 66f0fd7d05b3fae8e9c12684b16240e2e9ae0add263018ec49e82564d8b1489a9564a3e8899a1a975a145500c5eee9afe002a9644b368bc0cf2c5206328d71fd
+  metadata.gz: bc1755b0fe3837399d9fb8143932ded28a711b0b36f257a7c98e7440c469ca065f63c5065058c84177ee1f15646b3b51058f8cc3188801470e016b3a023a2eff
+  data.tar.gz: f13e0d7d83c67818ed3d73ff6f4d423f99c18df6d9ff2118e651580668cc102484cbdbf2f4d74ab93360656b9285beb190d7291c952930d58ba6eb7cd2bf275f

data/lib/news_crawler/config.rb

@@ -21,6 +21,7 @@
 #++
 
 require 'simple_config'
+require 'yaml'
 
 module NewsCrawler
   class CrawlerConfig
@@ -32,17 +33,32 @@ module NewsCrawler
     def self.load_application_config(file = CrawlerConfig::DEFAULT_CONFIG)
       if ((file != DEFAULT_CONFIG) || (@app_loaded != true))
         @app_loaded = true
-        SimpleConfig.for :application do
-          load file
-        end
+        merge_config(:application, file)
       end
     end
 
     def self.load_samedomainselector_config(file = CrawlerConfig::DEFAULT_SDS_CONFIG)
       if ((file != DEFAULT_SDS_CONFIG) || (@sds_loaded != true))
         @sds_loaded = true
-        SimpleConfig.for :same_domain_selector do
-          load file
+        merge_config(:same_domain_selector, file)
+      end
+    end
+
+    def self.merge_config(mod, file)
+      conf = YAML.load_file(file)
+      conf.each do | key, val |
+        if val.is_a? Hash
+          val.each do | k1, v1 |
+            SimpleConfig.for mod do
+              group key do
+                set k1, v1
+              end
+            end
+          end
+        else
+          SimpleConfig.for mod do
+            set key, val
+          end
         end
       end
     end

data/lib/news_crawler/link_selector/same_domain_selector.rb

@@ -119,10 +119,12 @@ module NewsCrawler
           return false
         end
 
-        exclude_group.to_hash.keys.each do | url_e |
-          if url_domain.to_s.end_with? url_e.to_s
-            exclude_list = config.exclude.get(url_e)
-            break
+        unless exclude_group.nil?
+          exclude_group.to_hash.keys.each do | url_e |
+            if url_domain.to_s.end_with? url_e.to_s
+              exclude_list = config.exclude.get(url_e)
+              break
+            end
           end
         end
 

data/lib/news_crawler/storage/raw_data/mongo_storage.rb

@@ -23,6 +23,7 @@
 require 'mongo'
 require 'simple_config'
 require 'news_crawler/storage/raw_data/raw_data_engine'
+require 'news_crawler/nc_logger'
 
 
 module NewsCrawler
@@ -46,6 +47,7 @@ module NewsCrawler
         # @param [ String ] url
         # @param [ String ] body
         def add(url, body)
+          body.encode!('utf-8', :invalid => :replace, :undef => :replace)
           @coll.update({:url   => url},
                        {:$set  => {:body => body}},
                        {:upsert => true})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: news_crawler
 version: !ruby/object:Gem::Version
-  version: 0.0.3.pre.1
+  version: 0.0.3.pre.3
 platform: ruby
 authors:
 - Hà Quang Dương
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-23 00:00:00.000000000 Z
+date: 2013-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo